From cb53bbb7b09d551fe276b59abe38a57694e091dd Mon Sep 17 00:00:00 2001 From: fibersel <0583463@gmail.com> Date: Thu, 6 May 2021 14:57:22 +0300 Subject: [PATCH 01/44] add experimental codecs flag, add integration test for experimental codecs --- programs/compressor/Compressor.cpp | 37 ++++++ src/CMakeLists.txt | 24 ++++ src/Client/Connection.cpp | 2 +- src/Compression/CompressionCodecDensity.cpp | 106 ++++++++++++++++++ src/Compression/CompressionCodecDensity.h | 35 ++++++ src/Compression/CompressionCodecLZSSE2.cpp | 91 +++++++++++++++ src/Compression/CompressionCodecLZSSE2.h | 33 ++++++ src/Compression/CompressionCodecLZSSE4.cpp | 91 +++++++++++++++ src/Compression/CompressionCodecLZSSE4.h | 33 ++++++ src/Compression/CompressionCodecLZSSE8.cpp | 90 +++++++++++++++ src/Compression/CompressionCodecLZSSE8.h | 33 ++++++ src/Compression/CompressionCodecLizard.cpp | 88 +++++++++++++++ src/Compression/CompressionCodecLizard.h | 34 ++++++ src/Compression/CompressionFactory.cpp | 46 +++++++- src/Compression/CompressionFactory.h | 10 +- src/Compression/CompressionInfo.h | 10 +- src/Core/Settings.h | 1 + src/Interpreters/InterpreterCreateQuery.cpp | 3 +- src/Server/TCPHandler.cpp | 2 +- src/Storages/AlterCommands.cpp | 8 +- src/Storages/ColumnsDescription.cpp | 2 +- .../DistributedBlockOutputStream.cpp | 2 +- src/Storages/TTLDescription.cpp | 2 +- .../test_non_default_compression/test.py | 19 +++- 24 files changed, 781 insertions(+), 21 deletions(-) create mode 100644 src/Compression/CompressionCodecDensity.cpp create mode 100644 src/Compression/CompressionCodecDensity.h create mode 100644 src/Compression/CompressionCodecLZSSE2.cpp create mode 100644 src/Compression/CompressionCodecLZSSE2.h create mode 100644 src/Compression/CompressionCodecLZSSE4.cpp create mode 100644 src/Compression/CompressionCodecLZSSE4.h create mode 100644 src/Compression/CompressionCodecLZSSE8.cpp create mode 100644 src/Compression/CompressionCodecLZSSE8.h create mode 100644 src/Compression/CompressionCodecLizard.cpp create mode 100644 src/Compression/CompressionCodecLizard.h diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index d47372631fe..d5471a62afe 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -19,6 +20,8 @@ #include #include +#include +#include namespace DB { @@ -77,6 +80,12 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) ("block-size,b", po::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") ("hc", "use LZ4HC instead of LZ4") ("zstd", "use ZSTD instead of LZ4") + ("lizard", "use Lizard instread of LZ4") + ("lzsse2", "use lzsse2 instread of LZ4") + ("lzsse4", "use lzsse4 instread of LZ4") + ("lzsse8", "use lzsse8 instread of LZ4") + ("density", "use Density instread of LZ4") + ("param", po::value(), "extra params for compresion algorithm") ("codec", po::value>()->multitoken(), "use codecs combination instead of LZ4") ("level", po::value(), "compression level for codecs specified via flags") ("none", "use no compression instead of LZ4") @@ -103,6 +112,11 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) bool decompress = options.count("decompress"); bool use_lz4hc = options.count("hc"); bool use_zstd = options.count("zstd"); + bool use_lizard = options.count("lizard"); + bool use_lzsse2 = options.count("lzsse2"); + bool use_lzsse4 = options.count("lzsse4"); + bool use_lzsse8 = options.count("lzsse8"); + bool use_density = options.count("density"); bool stat_mode = options.count("stat"); bool use_none = options.count("none"); unsigned block_size = options["block-size"].as(); @@ -110,6 +124,10 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) if (options.count("codec")) codecs = options["codec"].as>(); + std::optional param; + if (options.count("param")) + param = options["param"].as(); + if ((use_lz4hc || use_zstd || use_none) && !codecs.empty()) throw Exception("Wrong options, codec flags like --zstd and --codec options are mutually exclusive", ErrorCodes::BAD_ARGUMENTS); @@ -122,6 +140,16 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) method_family = "LZ4HC"; else if (use_zstd) method_family = "ZSTD"; + else if (use_lizard) + method_family = "Lizard"; + else if (use_lzsse2) + method_family = "LZSSE2"; + else if (use_lzsse4) + method_family = "LZSSE4"; + else if (use_lzsse8) + method_family = "LZSSE8"; + else if (use_density) + method_family = "Density"; else if (use_none) method_family = "NONE"; @@ -137,6 +165,8 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) std::string codecs_line = boost::algorithm::join(codecs, ","); auto ast = parseQuery(codec_parser, "(" + codecs_line + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); codec = CompressionCodecFactory::instance().get(ast, nullptr); + } else if (param.has_value()) { + codec = CompressionCodecFactory::instance().get(method_family, level, param); } else codec = CompressionCodecFactory::instance().get(method_family, level); @@ -155,6 +185,9 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) else wb = std::make_unique(STDOUT_FILENO); + + struct tms tv1, tv2; + times(&tv1); if (stat_mode) { /// Output statistic for compressed file. @@ -185,6 +218,10 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) CompressedWriteBuffer to(*wb, codec, block_size); copyData(*rb, to); } + times(&tv2); + + int tics_per_second = sysconf(_SC_CLK_TCK); + std::cerr << static_cast(tv2.tms_utime - tv1.tms_utime) / tics_per_second << std::endl; } catch (...) { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 69a84dbeb2c..ea8525212cc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -377,6 +377,30 @@ if (XZ_LIBRARY) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${XZ_INCLUDE_DIR}) endif() +set (LIZARD_LIBRARY lizard) +set (LIZARD_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/lizard/lib/) +if (LIZARD_LIBRARY) + dbms_target_link_libraries(PRIVATE ${LIZARD_LIBRARY}) + #target_link_libraries (clickhouse_compression PUBLIC ${LZMA_LIBRARY}) + #target_include_directories (clickhouse_compression SYSTEM BEFORE PUBLIC ${}) +endif() + +set (DENSITY_LIBRARY density) +set (DENSITY_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/density/src) +if (DENSITY_LIBRARY) + dbms_target_link_libraries(PRIVATE ${DENSITY_LIBRARY}) + #target_link_libraries (clickhouse_compression PUBLIC ${LZMA_LIBRARY}) + #target_include_directories (clickhouse_compression SYSTEM BEFORE PUBLIC ${}) +endif() + +set (LZSSE_LIBRARY lzsse) +set (LZSSE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/lzsse/) +if (LZSSE_LIBRARY) + dbms_target_link_libraries(PRIVATE ${LZSSE_LIBRARY}) + #target_link_libraries (clickhouse_compression PUBLIC ${LZMA_LIBRARY}) + #target_include_directories (clickhouse_compression SYSTEM BEFORE PUBLIC ${}) +endif() + if (USE_ICU) dbms_target_link_libraries (PRIVATE ${ICU_LIBRARIES}) dbms_target_include_directories (SYSTEM PRIVATE ${ICU_INCLUDE_DIRS}) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index d26f7454bcc..87f768d7e75 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -424,7 +424,7 @@ void Connection::sendQuery( if (method == "ZSTD") level = settings->network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs); + CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs); compression_codec = CompressionCodecFactory::instance().get(method, level); } else diff --git a/src/Compression/CompressionCodecDensity.cpp b/src/Compression/CompressionCodecDensity.cpp new file mode 100644 index 00000000000..841638ac6ed --- /dev/null +++ b/src/Compression/CompressionCodecDensity.cpp @@ -0,0 +1,106 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CANNOT_COMPRESS; + extern const int CANNOT_DECOMPRESS; + extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; + extern const int ILLEGAL_CODEC_PARAMETER; +} + +CompressionCodecDensity::CompressionCodecDensity(DENSITY_ALGORITHM algo_) : algo(algo_) +{ + setCodecDescription("Density", {std::make_shared(static_cast(algo))}); +} + +uint8_t CompressionCodecDensity::getMethodByte() const +{ + return static_cast(CompressionMethodByte::Density); +} + +void CompressionCodecDensity::updateHash(SipHash & hash) const +{ + getCodecDesc()->updateTreeHash(hash); +} + +UInt32 CompressionCodecDensity::getMaxCompressedDataSize(UInt32 uncompressed_size) const +{ + return density_compress_safe_size(uncompressed_size); +} + +UInt32 CompressionCodecDensity::doCompressData(const char * source, UInt32 source_size, char * dest) const +{ + density_processing_result res = density_compress(reinterpret_cast(source), source_size, reinterpret_cast(dest), density_compress_safe_size(source_size), algo); + if (res.state != DENSITY_STATE_OK) + throw Exception("Cannot compress block with Density; ", ErrorCodes::CANNOT_COMPRESS); + return res.bytesWritten; +} + +void CompressionCodecDensity::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const +{ + density_processing_result res = density_decompress(reinterpret_cast(source), source_size, reinterpret_cast(dest), density_decompress_safe_size(uncompressed_size)); + if (res.state != DENSITY_STATE_OK) + throw Exception("Cannot decompress block with Density; ", ErrorCodes::CANNOT_DECOMPRESS); +} + +void registerCodecDensity(CompressionCodecFactory & factory) +{ + UInt8 method_code = UInt8(CompressionMethodByte::Density); + factory.registerCompressionCodec( + "Density", + method_code, + [&](const ASTPtr & arguments) -> CompressionCodecPtr + { + DENSITY_ALGORITHM algo = CompressionCodecDensity::DENSITY_DEFAULT_ALGO; + //std::cerr << arguments << std::endl; + if (arguments && !arguments->children.empty()) + { + if (arguments->children.size() != 1) + throw Exception( + "Deisnty codec must have 1 parameter, given " + std::to_string(arguments->children.size()), + ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE); + + const auto children = arguments->children; + + const auto * algo_literal = children[0]->as(); + if (!algo_literal) + throw Exception("Density codec argument must be string (algorithm)", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + + + if (algo_literal->value.getType() == Field::Types::Which::UInt64) { + const auto algorithm = algo_literal->value.safeGet(); + if (algorithm == 3) { + algo = DENSITY_ALGORITHM_LION; + } else if (algorithm == 2) { + algo = DENSITY_ALGORITHM_CHEETAH; + } else if (algorithm == 1) { + algo = DENSITY_ALGORITHM_CHAMELEON; + } else { + throw Exception("Density codec argument may be LION, CHAMELEON, CHEETAH", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + } + } else { + const auto algorithm = algo_literal->value.safeGet(); + if (algorithm == "LION") { + algo = DENSITY_ALGORITHM_LION; + } else if (algorithm == "CHAMELEON") { + algo = DENSITY_ALGORITHM_CHAMELEON; + } else if (algorithm == "CHEETAH") { + algo = DENSITY_ALGORITHM_CHEETAH; + } else { + throw Exception("Density codec argument may be LION, CHAMELEON, CHEETAH", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + } + } + } + + return std::make_shared(algo); + }); +} + +} diff --git a/src/Compression/CompressionCodecDensity.h b/src/Compression/CompressionCodecDensity.h new file mode 100644 index 00000000000..cfc8bfbd76a --- /dev/null +++ b/src/Compression/CompressionCodecDensity.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class CompressionCodecDensity : public ICompressionCodec +{ +public: + static constexpr auto DENSITY_DEFAULT_ALGO = DENSITY_ALGORITHM_CHAMELEON; // by default aim on speed + + CompressionCodecDensity(DENSITY_ALGORITHM algo_); + + uint8_t getMethodByte() const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; + + void updateHash(SipHash & hash) const override; + +protected: + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + + bool isCompression() const override { return true; } + + bool isGenericCompression() const override { return true; } + +private: + const DENSITY_ALGORITHM algo; +}; + +} \ No newline at end of file diff --git a/src/Compression/CompressionCodecLZSSE2.cpp b/src/Compression/CompressionCodecLZSSE2.cpp new file mode 100644 index 00000000000..215b93e6721 --- /dev/null +++ b/src/Compression/CompressionCodecLZSSE2.cpp @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CANNOT_COMPRESS; + extern const int CANNOT_DECOMPRESS; + extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; + extern const int ILLEGAL_CODEC_PARAMETER; +} + +CompressionCodecLZSSE2::CompressionCodecLZSSE2(int level_) : level(level_) +{ + + setCodecDescription("LZSSE2", {std::make_shared(static_cast(level))}); +} + +uint8_t CompressionCodecLZSSE2::getMethodByte() const +{ + return static_cast(CompressionMethodByte::LZSSE2); +} + +void CompressionCodecLZSSE2::updateHash(SipHash & hash) const +{ + getCodecDesc()->updateTreeHash(hash); +} + +UInt32 CompressionCodecLZSSE2::getMaxCompressedDataSize(UInt32 uncompressed_size) const +{ + return uncompressed_size; +} + +UInt32 CompressionCodecLZSSE2::doCompressData(const char * source, UInt32 source_size, char * dest) const +{ + UInt32 res; + LZSSE2_OptimalParseState* state = LZSSE2_MakeOptimalParseState(source_size); + res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level); + LZSSE2_FreeOptimalParseState(state); + + if (res == 0) + throw Exception("Cannot compress block with LZSSE2; ", ErrorCodes::CANNOT_COMPRESS); + + return res; +} + +void CompressionCodecLZSSE2::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const +{ + UInt32 res; + res = LZSSE2_Decompress(source, source_size, dest, uncompressed_size); + + if (res < uncompressed_size) + throw Exception("Cannot decompress block with LZSSE2; ", ErrorCodes::CANNOT_DECOMPRESS); +} + +void registerCodecLZSSE2(CompressionCodecFactory & factory) +{ + UInt8 method_code = UInt8(CompressionMethodByte::LZSSE2); + factory.registerCompressionCodec( + "LZSSE2", + method_code, + [&](const ASTPtr & arguments) -> CompressionCodecPtr + { + int level = 1; + if (arguments && !arguments->children.empty()) + { + if (arguments->children.size() != 1) + throw Exception( + "LZSSE2 codec must have 1 parameter, given " + std::to_string(arguments->children.size()), + ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE); + + const auto children = arguments->children; + const auto * level_literal = children[0]->as(); + if (!level_literal) + throw Exception("LZSSE2 first codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + + level = level_literal->value.safeGet(); + + } + + return std::make_shared(level); + }); +} + +} diff --git a/src/Compression/CompressionCodecLZSSE2.h b/src/Compression/CompressionCodecLZSSE2.h new file mode 100644 index 00000000000..d26bb3e2cf2 --- /dev/null +++ b/src/Compression/CompressionCodecLZSSE2.h @@ -0,0 +1,33 @@ +#pragma once + +#include + + +namespace DB +{ +class CompressionCodecLZSSE2 : public ICompressionCodec +{ +public: + + CompressionCodecLZSSE2(int level_); + + uint8_t getMethodByte() const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; + + void updateHash(SipHash & hash) const override; + +protected: + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + + bool isCompression() const override { return true; } + + bool isGenericCompression() const override { return true; } + +private: + const int level; +}; + +} \ No newline at end of file diff --git a/src/Compression/CompressionCodecLZSSE4.cpp b/src/Compression/CompressionCodecLZSSE4.cpp new file mode 100644 index 00000000000..9b687d866a1 --- /dev/null +++ b/src/Compression/CompressionCodecLZSSE4.cpp @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CANNOT_COMPRESS; + extern const int CANNOT_DECOMPRESS; + extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; + extern const int ILLEGAL_CODEC_PARAMETER; +} + +CompressionCodecLZSSE4::CompressionCodecLZSSE4(int level_) : level(level_) +{ + + setCodecDescription("LZSSE4", {std::make_shared(static_cast(level))}); +} + +uint8_t CompressionCodecLZSSE4::getMethodByte() const +{ + return static_cast(CompressionMethodByte::LZSSE4); +} + +void CompressionCodecLZSSE4::updateHash(SipHash & hash) const +{ + getCodecDesc()->updateTreeHash(hash); +} + +UInt32 CompressionCodecLZSSE4::getMaxCompressedDataSize(UInt32 uncompressed_size) const +{ + return uncompressed_size; +} + +UInt32 CompressionCodecLZSSE4::doCompressData(const char * source, UInt32 source_size, char * dest) const +{ + UInt32 res; + LZSSE4_OptimalParseState* state = LZSSE4_MakeOptimalParseState(source_size); + res = LZSSE4_CompressOptimalParse(state, source, source_size, dest, source_size, level); + LZSSE4_FreeOptimalParseState(state); + + if (res == 0) + throw Exception("Cannot compress block with LZSSE4; ", ErrorCodes::CANNOT_COMPRESS); + + return res; +} + +void CompressionCodecLZSSE4::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const +{ + UInt32 res; + res = LZSSE4_Decompress(source, source_size, dest, uncompressed_size); + + if (res < uncompressed_size) + throw Exception("Cannot decompress block with LZSSE4; ", ErrorCodes::CANNOT_DECOMPRESS); +} + +void registerCodecLZSSE4(CompressionCodecFactory & factory) +{ + UInt8 method_code = UInt8(CompressionMethodByte::LZSSE4); + factory.registerCompressionCodec( + "LZSSE4", + method_code, + [&](const ASTPtr & arguments) -> CompressionCodecPtr + { + int level = 1; + if (arguments && !arguments->children.empty()) + { + if (arguments->children.size() != 1) + throw Exception( + "LZSSE4 codec must have 1 parameter, given " + std::to_string(arguments->children.size()), + ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE); + + const auto children = arguments->children; + const auto * level_literal = children[0]->as(); + if (!level_literal) + throw Exception("LZSSE4 first codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + + level = level_literal->value.safeGet(); + + } + + return std::make_shared(level); + }); +} + +} diff --git a/src/Compression/CompressionCodecLZSSE4.h b/src/Compression/CompressionCodecLZSSE4.h new file mode 100644 index 00000000000..ed1cc8fb26b --- /dev/null +++ b/src/Compression/CompressionCodecLZSSE4.h @@ -0,0 +1,33 @@ +#pragma once + +#include + + +namespace DB +{ +class CompressionCodecLZSSE4 : public ICompressionCodec +{ +public: + + CompressionCodecLZSSE4(int level_); + + uint8_t getMethodByte() const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; + + void updateHash(SipHash & hash) const override; + +protected: + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + + bool isCompression() const override { return true; } + + bool isGenericCompression() const override { return true; } + +private: + const int level; +}; + +} \ No newline at end of file diff --git a/src/Compression/CompressionCodecLZSSE8.cpp b/src/Compression/CompressionCodecLZSSE8.cpp new file mode 100644 index 00000000000..04e73121fc6 --- /dev/null +++ b/src/Compression/CompressionCodecLZSSE8.cpp @@ -0,0 +1,90 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CANNOT_COMPRESS; + extern const int CANNOT_DECOMPRESS; + extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; + extern const int ILLEGAL_CODEC_PARAMETER; +} + +CompressionCodecLZSSE8::CompressionCodecLZSSE8(int level_) : level(level_) +{ + setCodecDescription("LZSSE8", {std::make_shared(static_cast(level))}); +} + +uint8_t CompressionCodecLZSSE8::getMethodByte() const +{ + return static_cast(CompressionMethodByte::LZSSE8); +} + +void CompressionCodecLZSSE8::updateHash(SipHash & hash) const +{ + getCodecDesc()->updateTreeHash(hash); +} + +UInt32 CompressionCodecLZSSE8::getMaxCompressedDataSize(UInt32 uncompressed_size) const +{ + return uncompressed_size; +} + +UInt32 CompressionCodecLZSSE8::doCompressData(const char * source, UInt32 source_size, char * dest) const +{ + UInt32 res; + LZSSE8_OptimalParseState* state = LZSSE8_MakeOptimalParseState(source_size); + res = LZSSE8_CompressOptimalParse(state, source, source_size, dest, source_size, level); + LZSSE8_FreeOptimalParseState(state); + + if (res == 0) + throw Exception("Cannot compress block with LZSSE; ", ErrorCodes::CANNOT_COMPRESS); + + return res; +} + +void CompressionCodecLZSSE8::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const +{ + UInt32 res; + res = LZSSE8_Decompress(source, source_size, dest, uncompressed_size); + + if (res < uncompressed_size) + throw Exception("Cannot decompress block with LZSSE; ", ErrorCodes::CANNOT_DECOMPRESS); +} + +void registerCodecLZSSE8(CompressionCodecFactory & factory) +{ + UInt8 method_code = UInt8(CompressionMethodByte::LZSSE8); + factory.registerCompressionCodec( + "LZSSE8", + method_code, + [&](const ASTPtr & arguments) -> CompressionCodecPtr + { + int level = 1; + if (arguments && !arguments->children.empty()) + { + if (arguments->children.size() != 1) + throw Exception( + "LZSSE8 codec must have 1 parameter, given " + std::to_string(arguments->children.size()), + ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE); + + const auto children = arguments->children; + const auto * level_literal = children[0]->as(); + if (!level_literal) + throw Exception("LZSSE8 first codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + + level = level_literal->value.safeGet(); + + } + + return std::make_shared(level); + }); +} + +} diff --git a/src/Compression/CompressionCodecLZSSE8.h b/src/Compression/CompressionCodecLZSSE8.h new file mode 100644 index 00000000000..c9d006ca13b --- /dev/null +++ b/src/Compression/CompressionCodecLZSSE8.h @@ -0,0 +1,33 @@ +#pragma once + +#include + + +namespace DB +{ +class CompressionCodecLZSSE8 : public ICompressionCodec +{ +public: + + CompressionCodecLZSSE8(int level_); + + uint8_t getMethodByte() const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; + + void updateHash(SipHash & hash) const override; + +protected: + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + + bool isCompression() const override { return true; } + + bool isGenericCompression() const override { return true; } + +private: + const int level; +}; + +} \ No newline at end of file diff --git a/src/Compression/CompressionCodecLizard.cpp b/src/Compression/CompressionCodecLizard.cpp new file mode 100644 index 00000000000..a4106965bd5 --- /dev/null +++ b/src/Compression/CompressionCodecLizard.cpp @@ -0,0 +1,88 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CANNOT_COMPRESS; + extern const int CANNOT_DECOMPRESS; + extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; + extern const int ILLEGAL_CODEC_PARAMETER; +} + +CompressionCodecLizard::CompressionCodecLizard(int level_) : level(level_) +{ + setCodecDescription("Lizard", {std::make_shared(static_cast(level))}); +} + +uint8_t CompressionCodecLizard::getMethodByte() const +{ + return static_cast(CompressionMethodByte::Lizard); +} + +void CompressionCodecLizard::updateHash(SipHash & hash) const +{ + getCodecDesc()->updateTreeHash(hash); +} + +UInt32 CompressionCodecLizard::getMaxCompressedDataSize(UInt32 uncompressed_size) const +{ + return Lizard_compressBound(uncompressed_size); +} + +UInt32 CompressionCodecLizard::doCompressData(const char * source, UInt32 source_size, char * dest) const +{ + int res = Lizard_compress(source, dest, source_size, Lizard_compressBound(source_size), level); + + if (res == 0) + throw Exception("Cannot compress block with Lizard; ", ErrorCodes::CANNOT_COMPRESS); + return res; +} + +void CompressionCodecLizard::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const +{ + int res = Lizard_decompress_safe(source, dest, source_size, uncompressed_size); + + if (res < 0) + throw Exception("Cannot compress block with Lizard; ", ErrorCodes::CANNOT_DECOMPRESS); +} + +void registerCodecLizard(CompressionCodecFactory & factory) +{ + UInt8 method_code = UInt8(CompressionMethodByte::Lizard); + factory.registerCompressionCodec( + "Lizard", + method_code, + [&](const ASTPtr & arguments) -> CompressionCodecPtr + { + int level = CompressionCodecLizard::LIZARD_DEFAULT_LEVEL; + if (arguments && !arguments->children.empty()) + { + if (arguments->children.size() > 1) + throw Exception( + "Lizard codec must have 1 parameter, given " + std::to_string(arguments->children.size()), + ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE); + + const auto children = arguments->children; + const auto * literal = children[0]->as(); + if (!literal) + throw Exception("Lizard codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + + level = literal->value.safeGet(); + // compression level will be truncated to LIZARD_MAX_CLEVEL if it is greater and to LIZARD_MIN_CLEVEL if it is less + //if (level > 1)//ZSTD_maxCLevel()) + // throw Exception("Lizard codec can't have level more that " + toString(1/*ZSTD_maxCLevel()*/) + ", given " + toString(level), ErrorCodes::ILLEGAL_CODEC_PARAMETER); + } + + return std::make_shared(level); + }); +} + +} diff --git a/src/Compression/CompressionCodecLizard.h b/src/Compression/CompressionCodecLizard.h new file mode 100644 index 00000000000..95bef7ab39e --- /dev/null +++ b/src/Compression/CompressionCodecLizard.h @@ -0,0 +1,34 @@ +#pragma once + +#include + + +namespace DB +{ +class CompressionCodecLizard : public ICompressionCodec +{ +public: + static constexpr auto LIZARD_DEFAULT_LEVEL = 1; + + CompressionCodecLizard(int level_); + + uint8_t getMethodByte() const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; + + void updateHash(SipHash & hash) const override; + +protected: + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + + bool isCompression() const override { return true; } + + bool isGenericCompression() const override { return true; } + +private: + const int level; +}; + +} \ No newline at end of file diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index 0ff9797aeaf..9ca64e36468 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -44,7 +44,24 @@ CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std } } -void CompressionCodecFactory::validateCodec(const String & family_name, std::optional level, bool sanity_check) const +CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std::optional level, std::optional param) const +{ + if (level && param) + { + auto level_literal = std::make_shared(static_cast(*level)); + auto param_literal = std::make_shared(static_cast(*param)); + return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), level_literal, param_literal)), {}); + } + else if (param) + { + auto param_literal = std::make_shared(static_cast(*param)); + return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), param_literal)), {}); + } else { + return get(family_name, level); + } +} + +void CompressionCodecFactory::validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs) const { if (family_name.empty()) throw Exception("Compression codec name cannot be empty", ErrorCodes::BAD_ARGUMENTS); @@ -52,16 +69,16 @@ void CompressionCodecFactory::validateCodec(const String & family_name, std::opt if (level) { auto literal = std::make_shared(static_cast(*level)); - validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), {}, sanity_check); + validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), {}, sanity_check, allow_experimental_codecs); } else { auto identifier = std::make_shared(Poco::toUpper(family_name)); - validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), {}, sanity_check); + validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), {}, sanity_check, allow_experimental_codecs); } } -ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const IDataType * column_type, bool sanity_check) const +ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const IDataType * column_type, bool sanity_check, bool allow_experimental_codecs) const { if (const auto * func = ast->as()) { @@ -90,6 +107,16 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr else throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE); + if (sanity_check && !allow_experimental_codecs) { + if (codec_family_name == "Lizard" || + codec_family_name == "Density" || + codec_family_name == "LZSSE2" || + codec_family_name == "LZSSE4" || + codec_family_name == "LZSSE8") { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Experimental codecs Lizard, Density and LZSSE* are not allowed, please enable allow_experimental_codecs flag."); + } + } /// Default codec replaced with current default codec which may depend on different /// settings (and properties of data) in runtime. CompressionCodecPtr result_codec; @@ -172,6 +199,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS); } + /// For columns with nested types like Tuple(UInt32, UInt64) we /// obviously cannot substitute parameters for codecs which depend on /// data type, because for the first column Delta(4) is suitable and @@ -318,6 +346,11 @@ void registerCodecT64(CompressionCodecFactory & factory); void registerCodecDoubleDelta(CompressionCodecFactory & factory); void registerCodecGorilla(CompressionCodecFactory & factory); void registerCodecMultiple(CompressionCodecFactory & factory); +void registerCodecLizard(CompressionCodecFactory & factory); +void registerCodecDensity(CompressionCodecFactory & factory); +void registerCodecLZSSE2(CompressionCodecFactory & factory); +void registerCodecLZSSE4(CompressionCodecFactory & factory); +void registerCodecLZSSE8(CompressionCodecFactory & factory); CompressionCodecFactory::CompressionCodecFactory() { @@ -330,6 +363,11 @@ CompressionCodecFactory::CompressionCodecFactory() registerCodecDoubleDelta(*this); registerCodecGorilla(*this); registerCodecMultiple(*this); + registerCodecLizard(*this); + registerCodecDensity(*this); + registerCodecLZSSE2(*this); + registerCodecLZSSE4(*this); + registerCodecLZSSE8(*this); default_codec = get("LZ4", {}); } diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h index dc06e853898..34dd8633611 100644 --- a/src/Compression/CompressionFactory.h +++ b/src/Compression/CompressionFactory.h @@ -38,16 +38,16 @@ public: CompressionCodecPtr getDefaultCodec() const; /// Validate codecs AST specified by user and parses codecs description (substitute default parameters) - ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const IDataType * column_type, bool sanity_check) const; + ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const IDataType * column_type, bool sanity_check, bool allow_experimental_codecs) const; /// Just wrapper for previous method. - ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check) const + ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs) const { - return validateCodecAndGetPreprocessedAST(ast, column_type.get(), sanity_check); + return validateCodecAndGetPreprocessedAST(ast, column_type.get(), sanity_check, allow_experimental_codecs); } /// Validate codecs AST specified by user - void validateCodec(const String & family_name, std::optional level, bool sanity_check) const; + void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs) const; /// Get codec by AST and possible column_type. Some codecs can use /// information about type to improve inner settings, but every codec should @@ -72,6 +72,8 @@ public: /// For backward compatibility with config settings CompressionCodecPtr get(const String & family_name, std::optional level) const; + CompressionCodecPtr get(const String & family_name, std::optional level, std::optional param) const; + /// Register codec with parameters and column type void registerCompressionCodecWithType(const String & family_name, std::optional byte_code, CreatorWithType creator); /// Register codec with parameters diff --git a/src/Compression/CompressionInfo.h b/src/Compression/CompressionInfo.h index 290ebaeb504..d1d4ad4618f 100644 --- a/src/Compression/CompressionInfo.h +++ b/src/Compression/CompressionInfo.h @@ -41,8 +41,14 @@ enum class CompressionMethodByte : uint8_t Multiple = 0x91, Delta = 0x92, T64 = 0x93, - DoubleDelta = 0x94, - Gorilla = 0x95, + DoubleDelta = 0x94, + Gorilla = 0x95, + Lizard = 0x96, + Density = 0x97, + LZSSE2 = 0x98, + LZSSE4 = 0x99, + LZSSE8 = 0xa0, + }; } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 42a20441a2e..9ce06542d2c 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -238,6 +238,7 @@ class IColumn; M(Bool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \ M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \ + M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs(LZSSE*, Lizard, Density).", 0) \ M(UInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \ M(UInt64, query_profiler_real_time_period_ns, 1000000000, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(UInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index d45d02243fb..6da714ee6b9 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -437,6 +437,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( defaults_sample_block = validateColumnsDefaultsAndGetSampleBlock(default_expr_list, column_names_and_types, context_); bool sanity_check_compression_codecs = !attach && !context_->getSettingsRef().allow_suspicious_codecs; + bool allow_experimental_codecs = context_->getSettingsRef().allow_experimental_codecs; ColumnsDescription res; auto name_type_it = column_names_and_types.begin(); for (auto ast_it = columns_ast.children.begin(); ast_it != columns_ast.children.end(); ++ast_it, ++name_type_it) @@ -471,7 +472,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( if (col_decl.default_specifier == "ALIAS") throw Exception{"Cannot specify codec for column type ALIAS", ErrorCodes::BAD_ARGUMENTS}; column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - col_decl.codec, column.type, sanity_check_compression_codecs); + col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs); } if (col_decl.ttl) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 916f29ba1d4..e0abbd63b1d 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1398,7 +1398,7 @@ void TCPHandler::initBlockOutput(const Block & block) if (state.compression == Protocol::Compression::Enable) { - CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs); + CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs); state.maybe_compressed_out = std::make_shared( *out, CompressionCodecFactory::instance().get(method, level)); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index e3177c167c5..b7e7893487b 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -313,7 +313,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) column.comment = *comment; if (codec) - column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type, false); + column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type, false, true); column.ttl = ttl; @@ -354,7 +354,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) else { if (codec) - column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type ? data_type : column.type, false); + column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type ? data_type : column.type, false, true); if (comment) column.comment = *comment; @@ -907,7 +907,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, ContextPt ErrorCodes::BAD_ARGUMENTS}; if (command.codec) - CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs); + CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs); all_columns.add(ColumnDescription(column_name, command.data_type)); } @@ -927,7 +927,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, ContextPt ErrorCodes::NOT_IMPLEMENTED}; if (command.codec) - CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs); + CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs); auto column_default = all_columns.getDefault(column_name); if (column_default) { diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index 545911f1465..43a385bda1d 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -128,7 +128,7 @@ void ColumnDescription::readText(ReadBuffer & buf) comment = col_ast->comment->as().value.get(); if (col_ast->codec) - codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(col_ast->codec, type, false); + codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(col_ast->codec, type, false, true); if (col_ast->ttl) ttl = col_ast->ttl; diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index a4aa2779771..b91f5b10cf2 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -632,7 +632,7 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: if (compression_method == "ZSTD") compression_level = settings.network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs); + CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs); CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level); /// tmp directory is used to ensure atomicity of transactions diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index 95ea4f07f18..9bb7d0c22f2 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -289,7 +289,7 @@ TTLDescription TTLDescription::getTTLFromAST( { result.recompression_codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs); + ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs); } } diff --git a/tests/integration/test_non_default_compression/test.py b/tests/integration/test_non_default_compression/test.py index 03210e47081..678b2167595 100644 --- a/tests/integration/test_non_default_compression/test.py +++ b/tests/integration/test_non_default_compression/test.py @@ -17,7 +17,8 @@ node4 = cluster.add_instance('node4', user_configs=['configs/enable_uncompressed node5 = cluster.add_instance('node5', main_configs=['configs/zstd_compression_by_default.xml'], user_configs=['configs/enable_uncompressed_cache.xml', 'configs/allow_suspicious_codecs.xml']) - +node6 = cluster.add_instance('node6', main_configs=['configs/allow_experimental_codecs.xml'], + user_configs=['configs/allow_suspicious_codecs.xml']) @pytest.fixture(scope="module") def start_cluster(): @@ -137,3 +138,19 @@ def test_uncompressed_cache_plus_zstd_codec(start_cluster): assert node5.query( "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" + +def test_experimental_codecs(start_cluster): + node6.query(""" + CREATE TABLE compression_experimental_codecs ( + somedate Date CODEC(Lizard(12)), + id UInt64 CODEC(Density('LION')), + data String CODEC(LZSSE4(3)) + ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; + """) + + node6.query( + "INSERT INTO compression_experimental_codecs VALUES(toDate('2018-10-12'), 100000, '{}')".format( + 'c' * 10000)) + + assert node6.query( + "SELECT max(length(data)) from compression_experimental_codecs GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" From 9cfddff290e347b26558366a88951d5b3665ba66 Mon Sep 17 00:00:00 2001 From: fibersel <0583463@gmail.com> Date: Thu, 6 May 2021 15:08:58 +0300 Subject: [PATCH 02/44] add new codecs from contrib to cmake --- contrib/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 9eafec23f51..18da3ab5dd5 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -54,6 +54,10 @@ if (USE_INTERNAL_XZ_LIBRARY) add_subdirectory (xz) endif() +add_subdirectory (lizard-cmake) +add_subdirectory (density-cmake) +add_subdirectory (lzsse-cmake) + add_subdirectory (poco-cmake) add_subdirectory (croaring-cmake) From 7c9acb9f532709daa0a50ee3656086d6d670ba0a Mon Sep 17 00:00:00 2001 From: fibersel <0583463@gmail.com> Date: Thu, 6 May 2021 15:26:03 +0300 Subject: [PATCH 03/44] add dedicated cmake files for compression libraries --- contrib/density-cmake/CMakeLists.txt | 47 ++++++++++++++++++++++++++++ contrib/lizard-cmake/CMakeLists.txt | 19 +++++++++++ contrib/lzsse-cmake/CMakeLists.txt | 21 +++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 contrib/density-cmake/CMakeLists.txt create mode 100644 contrib/lizard-cmake/CMakeLists.txt create mode 100644 contrib/lzsse-cmake/CMakeLists.txt diff --git a/contrib/density-cmake/CMakeLists.txt b/contrib/density-cmake/CMakeLists.txt new file mode 100644 index 00000000000..d48bbbf05b9 --- /dev/null +++ b/contrib/density-cmake/CMakeLists.txt @@ -0,0 +1,47 @@ +SET (USE_INTERNAL_DENSITY_LIBRARY 1) +SET (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/density") + +# cd contrib/density +# find . -name '*.c' | grep -vP 'deprecated|legacy' | sort | sed 's/^\./ ${LIBRARY_DIR}/' +SET (Sources + ${LIBRARY_DIR}/src/algorithms/algorithms.c + ${LIBRARY_DIR}/src/algorithms/chameleon/core/chameleon_decode.c + ${LIBRARY_DIR}/src/algorithms/chameleon/core/chameleon_encode.c + ${LIBRARY_DIR}/src/algorithms/cheetah/core/cheetah_decode.c + ${LIBRARY_DIR}/src/algorithms/cheetah/core/cheetah_encode.c + ${LIBRARY_DIR}/src/algorithms/dictionaries.c + ${LIBRARY_DIR}/src/algorithms/lion/core/lion_decode.c + ${LIBRARY_DIR}/src/algorithms/lion/core/lion_encode.c + ${LIBRARY_DIR}/src/algorithms/lion/forms/lion_form_model.c + ${LIBRARY_DIR}/src/buffers/buffer.c + ${LIBRARY_DIR}/src/globals.c + ${LIBRARY_DIR}/src/structure/header.c +) + +# cd contrib/density +# find . -name '*.h' | grep -vP 'deprecated|legacy' | sort | sed 's/^\./ ${LIBRARY_DIR}/' +SET (Headers + ${LIBRARY_DIR}/src/algorithms/algorithms.h + ${LIBRARY_DIR}/src/algorithms/chameleon/chameleon.h + ${LIBRARY_DIR}/src/algorithms/chameleon/core/chameleon_decode.h + ${LIBRARY_DIR}/src/algorithms/chameleon/core/chameleon_encode.h + ${LIBRARY_DIR}/src/algorithms/chameleon/dictionary/chameleon_dictionary.h + ${LIBRARY_DIR}/src/algorithms/cheetah/cheetah.h + ${LIBRARY_DIR}/src/algorithms/cheetah/core/cheetah_decode.h + ${LIBRARY_DIR}/src/algorithms/cheetah/core/cheetah_encode.h + ${LIBRARY_DIR}/src/algorithms/cheetah/dictionary/cheetah_dictionary.h + ${LIBRARY_DIR}/src/algorithms/dictionaries.h + ${LIBRARY_DIR}/src/algorithms/lion/core/lion_decode.h + ${LIBRARY_DIR}/src/algorithms/lion/core/lion_encode.h + ${LIBRARY_DIR}/src/algorithms/lion/dictionary/lion_dictionary.h + ${LIBRARY_DIR}/src/algorithms/lion/forms/lion_form_model.h + ${LIBRARY_DIR}/src/algorithms/lion/lion.h + ${LIBRARY_DIR}/src/buffers/buffer.h + ${LIBRARY_DIR}/src/density_api.h + ${LIBRARY_DIR}/src/globals.h + ${LIBRARY_DIR}/src/structure/header.h +) + +ADD_LIBRARY(density ${Sources} ${Headers}) + +target_include_directories (density PUBLIC ${LIBRARY_DIR}) \ No newline at end of file diff --git a/contrib/lizard-cmake/CMakeLists.txt b/contrib/lizard-cmake/CMakeLists.txt new file mode 100644 index 00000000000..eaf63db0acd --- /dev/null +++ b/contrib/lizard-cmake/CMakeLists.txt @@ -0,0 +1,19 @@ +SET (USE_INTERNAL_LIZARD_LIBRARY 1) +SET (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/lizard") + +SET (Sources + "${LIBRARY_DIR}/lib/lizard_decompress.c" + "${LIBRARY_DIR}/lib/lizard_compress.c" + "${LIBRARY_DIR}/lib/lizard_frame.c" + "${LIBRARY_DIR}/lib/xxhash/xxhash.c" +) + +SET (Headers + "${LIBRARY_DIR}/lib/lizard_compress.h" + "${LIBRARY_DIR}/lib/lizard_common.h" + "${LIBRARY_DIR}/lib/lizard_frame.h" +) + +ADD_LIBRARY(lizard ${Sources} ${Headers}) + +target_include_directories (lizard PUBLIC ${LIBRARY_DIR}) \ No newline at end of file diff --git a/contrib/lzsse-cmake/CMakeLists.txt b/contrib/lzsse-cmake/CMakeLists.txt new file mode 100644 index 00000000000..a463089a5b6 --- /dev/null +++ b/contrib/lzsse-cmake/CMakeLists.txt @@ -0,0 +1,21 @@ +SET (USE_INTERNAL_LZSSE_LIBRARY 1) +SET (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/lzsse") + +SET (Sources + "${LIBRARY_DIR}/lzsse2/lzsse2.cpp" + "${LIBRARY_DIR}/lzsse4/lzsse4.cpp" + "${LIBRARY_DIR}/lzsse8/lzsse8.cpp" +) + +SET (Headers + "${LIBRARY_DIR}/lzsse2/lzsse2.h" + "${LIBRARY_DIR}/lzsse4/lzsse4.h" + "${LIBRARY_DIR}/lzsse8/lzsse8.h" + "${LIBRARY_DIR}/lzsse2/lzsse2_platform.h" + "${LIBRARY_DIR}/lzsse4/lzsse4_platform.h" + "${LIBRARY_DIR}/lzsse8/lzsse8_platform.h" +) + +ADD_LIBRARY(lzsse ${Sources} ${Headers}) + +target_include_directories (lzsse PUBLIC ${LIBRARY_DIR}) \ No newline at end of file From 9f7733567b8198558da9a2b4450d5c99050a2d28 Mon Sep 17 00:00:00 2001 From: fibersel <0583463@gmail.com> Date: Thu, 6 May 2021 15:36:49 +0300 Subject: [PATCH 04/44] extend modules --- .gitmodules | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitmodules b/.gitmodules index f7dcf5f4ac1..3e572e5280e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -227,3 +227,12 @@ [submodule "contrib/datasketches-cpp"] path = contrib/datasketches-cpp url = https://github.com/ClickHouse-Extras/datasketches-cpp.git +[submodule "contrib/lizard"] + path = contrib/lizard + url = https://github.com/inikep/lizard +[submodule "contrib/density"] + path = contrib/density + url = https://github.com/centaurean/density.git +[submodule "contrib/lzsse"] + path = contrib/lzsse + url = https://github.com/ConorStokes/LZSSE.git From a2f70306dff7fce1be79059713cfbd21c1720ed6 Mon Sep 17 00:00:00 2001 From: fibersel <0583463@gmail.com> Date: Thu, 6 May 2021 15:55:03 +0300 Subject: [PATCH 05/44] add contribs --- contrib/density/.gitignore | 2 + contrib/density/.gitmodules | 6 + contrib/density/.travis.yml | 272 +++ contrib/density/CHANGELOG.md | 168 ++ contrib/density/LICENSE.md | 28 + contrib/density/README.md | 261 +++ contrib/density/appveyor.yml | 37 + contrib/density/benchmark/src/benchmark.c | 345 ++++ contrib/density/benchmark/src/benchmark.h | 129 ++ contrib/density/msvc/Density.sln | 21 + contrib/density/msvc/benchmark.vcxproj | 78 + .../density/msvc/benchmark.vcxproj.filters | 63 + contrib/density/msvc/density.vcxproj | 93 + contrib/density/msvc/density.vcxproj.filters | 135 ++ contrib/density/src/algorithms/algorithms.c | 43 + contrib/density/src/algorithms/algorithms.h | 78 + .../src/algorithms/chameleon/chameleon.h | 70 + .../chameleon/core/chameleon_decode.c | 254 +++ .../chameleon/core/chameleon_decode.h | 53 + .../chameleon/core/chameleon_encode.c | 179 ++ .../chameleon/core/chameleon_encode.h | 53 + .../dictionary/chameleon_dictionary.h | 63 + .../density/src/algorithms/cheetah/cheetah.h | 73 + .../algorithms/cheetah/core/cheetah_decode.c | 266 +++ .../algorithms/cheetah/core/cheetah_decode.h | 54 + .../algorithms/cheetah/core/cheetah_encode.c | 202 +++ .../algorithms/cheetah/core/cheetah_encode.h | 54 + .../cheetah/dictionary/cheetah_dictionary.h | 70 + contrib/density/src/algorithms/dictionaries.c | 48 + contrib/density/src/algorithms/dictionaries.h | 45 + .../src/algorithms/lion/core/lion_decode.c | 327 ++++ .../src/algorithms/lion/core/lion_decode.h | 54 + .../src/algorithms/lion/core/lion_encode.c | 298 ++++ .../src/algorithms/lion/core/lion_encode.h | 54 + .../lion/dictionary/lion_dictionary.h | 73 + .../algorithms/lion/forms/lion_form_model.c | 153 ++ .../algorithms/lion/forms/lion_form_model.h | 96 + contrib/density/src/algorithms/lion/lion.h | 94 + contrib/density/src/buffers/buffer.c | 220 +++ contrib/density/src/buffers/buffer.h | 58 + contrib/density/src/density_api.h | 220 +++ contrib/density/src/globals.c | 47 + contrib/density/src/globals.h | 232 +++ contrib/density/src/structure/header.c | 57 + contrib/density/src/structure/header.h | 58 + contrib/lizard/.gitattributes | 21 + contrib/lizard/.gitignore | 37 + contrib/lizard/.travis.yml | 268 +++ contrib/lizard/LICENSE | 15 + contrib/lizard/NEWS | 41 + contrib/lizard/README.md | 95 + contrib/lizard/appveyor.yml | 143 ++ contrib/lizard/contrib/djgpp/LICENSE | 24 + contrib/lizard/contrib/djgpp/README.MD | 21 + contrib/lizard/doc/lizard_Block_format.md | 181 ++ contrib/lizard/doc/lizard_Frame_format.md | 312 ++++ contrib/lizard/examples/.gitignore | 8 + .../lizard/examples/HCStreaming_ringBuffer.c | 241 +++ contrib/lizard/examples/README.md | 8 + .../examples/blockStreaming_doubleBuffer.c | 203 +++ .../examples/blockStreaming_doubleBuffer.md | 100 ++ .../examples/blockStreaming_lineByLine.c | 210 +++ .../examples/blockStreaming_lineByLine.md | 122 ++ .../examples/blockStreaming_ringBuffer.c | 202 +++ contrib/lizard/examples/compress_functions.c | 303 ++++ contrib/lizard/examples/frameCompress.c | 169 ++ contrib/lizard/examples/printVersion.c | 13 + contrib/lizard/examples/simple_buffer.c | 91 + .../lizard/examples/streaming_api_basics.md | 87 + contrib/lizard/lib/.gitignore | 3 + contrib/lizard/lib/LICENSE | 25 + contrib/lizard/lib/README.md | 76 + contrib/lizard/lib/dll/liblizard.def | 19 + contrib/lizard/lib/entropy/README.md | 38 + contrib/lizard/lib/entropy/bitstream.h | 414 +++++ contrib/lizard/lib/entropy/entropy_common.c | 231 +++ contrib/lizard/lib/entropy/error_private.h | 115 ++ contrib/lizard/lib/entropy/error_public.h | 64 + contrib/lizard/lib/entropy/fse.h | 694 ++++++++ contrib/lizard/lib/entropy/fse_compress.c | 848 +++++++++ contrib/lizard/lib/entropy/fse_decompress.c | 329 ++++ contrib/lizard/lib/entropy/huf.h | 250 +++ contrib/lizard/lib/entropy/huf_compress.c | 612 +++++++ contrib/lizard/lib/entropy/huf_decompress.c | 885 ++++++++++ contrib/lizard/lib/entropy/mem.h | 372 ++++ contrib/lizard/lib/liblizard.pc.in | 15 + contrib/lizard/lib/lizard_common.h | 504 ++++++ contrib/lizard/lib/lizard_compress.c | 630 +++++++ contrib/lizard/lib/lizard_compress.h | 208 +++ contrib/lizard/lib/lizard_compress_liz.h | 301 ++++ contrib/lizard/lib/lizard_compress_lz4.h | 162 ++ contrib/lizard/lib/lizard_decompress.c | 372 ++++ contrib/lizard/lib/lizard_decompress.h | 152 ++ contrib/lizard/lib/lizard_decompress_liz.h | 220 +++ contrib/lizard/lib/lizard_decompress_lz4.h | 163 ++ contrib/lizard/lib/lizard_frame.c | 1362 ++++++++++++++ contrib/lizard/lib/lizard_frame.h | 303 ++++ contrib/lizard/lib/lizard_frame_static.h | 81 + contrib/lizard/lib/lizard_parser_fast.h | 196 +++ contrib/lizard/lib/lizard_parser_fastbig.h | 175 ++ contrib/lizard/lib/lizard_parser_fastsmall.h | 189 ++ contrib/lizard/lib/lizard_parser_hashchain.h | 369 ++++ .../lizard/lib/lizard_parser_lowestprice.h | 376 ++++ contrib/lizard/lib/lizard_parser_nochain.h | 318 ++++ contrib/lizard/lib/lizard_parser_optimal.h | 679 +++++++ contrib/lizard/lib/lizard_parser_pricefast.h | 250 +++ contrib/lizard/lib/xxhash/xxhash.c | 888 ++++++++++ contrib/lizard/lib/xxhash/xxhash.h | 293 +++ contrib/lizard/programs/.gitignore | 15 + contrib/lizard/programs/COPYING | 339 ++++ contrib/lizard/programs/README.md | 73 + contrib/lizard/programs/bench.c | 502 ++++++ contrib/lizard/programs/bench.h | 37 + contrib/lizard/programs/datagen.c | 188 ++ contrib/lizard/programs/datagen.h | 39 + contrib/lizard/programs/lizard.1 | 229 +++ contrib/lizard/programs/lizardcli.c | 581 ++++++ contrib/lizard/programs/lizardio.c | 896 ++++++++++ contrib/lizard/programs/lizardio.h | 101 ++ contrib/lizard/programs/platform.h | 145 ++ contrib/lizard/programs/util.h | 497 ++++++ contrib/lizard/tests/.gitignore | 13 + contrib/lizard/tests/COPYING | 339 ++++ contrib/lizard/tests/README.md | 15 + contrib/lizard/tests/datagencli.c | 192 ++ contrib/lizard/tests/frametest.c | 866 +++++++++ contrib/lizard/tests/fullbench.c | 732 ++++++++ contrib/lizard/tests/fuzzer.c | 1086 ++++++++++++ contrib/lizard/visual/.gitignore | 11 + contrib/lizard/visual/README.md | 53 + .../visual/VS2010/datagen/datagen.vcxproj | 165 ++ .../visual/VS2010/frametest/frametest.vcxproj | 177 ++ .../visual/VS2010/fullbench/fullbench.vcxproj | 175 ++ .../visual/VS2010/fuzzer/fuzzer.vcxproj | 174 ++ .../VS2010/liblizard-dll/liblizard-dll.rc | 51 + .../liblizard-dll/liblizard-dll.vcxproj | 179 ++ .../visual/VS2010/liblizard/liblizard.vcxproj | 176 ++ contrib/lizard/visual/VS2010/lizard.sln | 86 + contrib/lizard/visual/VS2010/lizard/lizard.rc | 51 + .../visual/VS2010/lizard/lizard.vcxproj | 190 ++ contrib/lzsse/.gitignore | 4 + contrib/lzsse/.travis.yml | 185 ++ contrib/lzsse/LICENSE | 23 + contrib/lzsse/README.md | 15 + contrib/lzsse/example/main.cpp | 389 ++++ contrib/lzsse/lzsse2/lzsse2.cpp | 1080 ++++++++++++ contrib/lzsse/lzsse2/lzsse2.h | 90 + contrib/lzsse/lzsse2/lzsse2_platform.h | 73 + contrib/lzsse/lzsse4/lzsse4.cpp | 1499 ++++++++++++++++ contrib/lzsse/lzsse4/lzsse4.h | 117 ++ contrib/lzsse/lzsse4/lzsse4_platform.h | 73 + contrib/lzsse/lzsse8/lzsse8.cpp | 1568 +++++++++++++++++ contrib/lzsse/lzsse8/lzsse8.h | 117 ++ contrib/lzsse/lzsse8/lzsse8_platform.h | 73 + contrib/lzsse/premake4.exe | Bin 0 -> 518144 bytes contrib/lzsse/premake4.lua | 26 + 156 files changed, 35085 insertions(+) create mode 100644 contrib/density/.gitignore create mode 100644 contrib/density/.gitmodules create mode 100644 contrib/density/.travis.yml create mode 100644 contrib/density/CHANGELOG.md create mode 100644 contrib/density/LICENSE.md create mode 100644 contrib/density/README.md create mode 100644 contrib/density/appveyor.yml create mode 100644 contrib/density/benchmark/src/benchmark.c create mode 100644 contrib/density/benchmark/src/benchmark.h create mode 100644 contrib/density/msvc/Density.sln create mode 100644 contrib/density/msvc/benchmark.vcxproj create mode 100644 contrib/density/msvc/benchmark.vcxproj.filters create mode 100644 contrib/density/msvc/density.vcxproj create mode 100644 contrib/density/msvc/density.vcxproj.filters create mode 100644 contrib/density/src/algorithms/algorithms.c create mode 100644 contrib/density/src/algorithms/algorithms.h create mode 100644 contrib/density/src/algorithms/chameleon/chameleon.h create mode 100644 contrib/density/src/algorithms/chameleon/core/chameleon_decode.c create mode 100644 contrib/density/src/algorithms/chameleon/core/chameleon_decode.h create mode 100644 contrib/density/src/algorithms/chameleon/core/chameleon_encode.c create mode 100644 contrib/density/src/algorithms/chameleon/core/chameleon_encode.h create mode 100644 contrib/density/src/algorithms/chameleon/dictionary/chameleon_dictionary.h create mode 100644 contrib/density/src/algorithms/cheetah/cheetah.h create mode 100644 contrib/density/src/algorithms/cheetah/core/cheetah_decode.c create mode 100644 contrib/density/src/algorithms/cheetah/core/cheetah_decode.h create mode 100644 contrib/density/src/algorithms/cheetah/core/cheetah_encode.c create mode 100644 contrib/density/src/algorithms/cheetah/core/cheetah_encode.h create mode 100644 contrib/density/src/algorithms/cheetah/dictionary/cheetah_dictionary.h create mode 100644 contrib/density/src/algorithms/dictionaries.c create mode 100644 contrib/density/src/algorithms/dictionaries.h create mode 100644 contrib/density/src/algorithms/lion/core/lion_decode.c create mode 100644 contrib/density/src/algorithms/lion/core/lion_decode.h create mode 100644 contrib/density/src/algorithms/lion/core/lion_encode.c create mode 100644 contrib/density/src/algorithms/lion/core/lion_encode.h create mode 100644 contrib/density/src/algorithms/lion/dictionary/lion_dictionary.h create mode 100644 contrib/density/src/algorithms/lion/forms/lion_form_model.c create mode 100644 contrib/density/src/algorithms/lion/forms/lion_form_model.h create mode 100644 contrib/density/src/algorithms/lion/lion.h create mode 100644 contrib/density/src/buffers/buffer.c create mode 100644 contrib/density/src/buffers/buffer.h create mode 100644 contrib/density/src/density_api.h create mode 100644 contrib/density/src/globals.c create mode 100644 contrib/density/src/globals.h create mode 100644 contrib/density/src/structure/header.c create mode 100644 contrib/density/src/structure/header.h create mode 100644 contrib/lizard/.gitattributes create mode 100644 contrib/lizard/.gitignore create mode 100644 contrib/lizard/.travis.yml create mode 100644 contrib/lizard/LICENSE create mode 100644 contrib/lizard/NEWS create mode 100644 contrib/lizard/README.md create mode 100644 contrib/lizard/appveyor.yml create mode 100644 contrib/lizard/contrib/djgpp/LICENSE create mode 100644 contrib/lizard/contrib/djgpp/README.MD create mode 100644 contrib/lizard/doc/lizard_Block_format.md create mode 100644 contrib/lizard/doc/lizard_Frame_format.md create mode 100644 contrib/lizard/examples/.gitignore create mode 100644 contrib/lizard/examples/HCStreaming_ringBuffer.c create mode 100644 contrib/lizard/examples/README.md create mode 100644 contrib/lizard/examples/blockStreaming_doubleBuffer.c create mode 100644 contrib/lizard/examples/blockStreaming_doubleBuffer.md create mode 100644 contrib/lizard/examples/blockStreaming_lineByLine.c create mode 100644 contrib/lizard/examples/blockStreaming_lineByLine.md create mode 100644 contrib/lizard/examples/blockStreaming_ringBuffer.c create mode 100644 contrib/lizard/examples/compress_functions.c create mode 100644 contrib/lizard/examples/frameCompress.c create mode 100644 contrib/lizard/examples/printVersion.c create mode 100644 contrib/lizard/examples/simple_buffer.c create mode 100644 contrib/lizard/examples/streaming_api_basics.md create mode 100644 contrib/lizard/lib/.gitignore create mode 100644 contrib/lizard/lib/LICENSE create mode 100644 contrib/lizard/lib/README.md create mode 100644 contrib/lizard/lib/dll/liblizard.def create mode 100644 contrib/lizard/lib/entropy/README.md create mode 100644 contrib/lizard/lib/entropy/bitstream.h create mode 100644 contrib/lizard/lib/entropy/entropy_common.c create mode 100644 contrib/lizard/lib/entropy/error_private.h create mode 100644 contrib/lizard/lib/entropy/error_public.h create mode 100644 contrib/lizard/lib/entropy/fse.h create mode 100644 contrib/lizard/lib/entropy/fse_compress.c create mode 100644 contrib/lizard/lib/entropy/fse_decompress.c create mode 100644 contrib/lizard/lib/entropy/huf.h create mode 100644 contrib/lizard/lib/entropy/huf_compress.c create mode 100644 contrib/lizard/lib/entropy/huf_decompress.c create mode 100644 contrib/lizard/lib/entropy/mem.h create mode 100644 contrib/lizard/lib/liblizard.pc.in create mode 100644 contrib/lizard/lib/lizard_common.h create mode 100644 contrib/lizard/lib/lizard_compress.c create mode 100644 contrib/lizard/lib/lizard_compress.h create mode 100644 contrib/lizard/lib/lizard_compress_liz.h create mode 100644 contrib/lizard/lib/lizard_compress_lz4.h create mode 100644 contrib/lizard/lib/lizard_decompress.c create mode 100644 contrib/lizard/lib/lizard_decompress.h create mode 100644 contrib/lizard/lib/lizard_decompress_liz.h create mode 100644 contrib/lizard/lib/lizard_decompress_lz4.h create mode 100644 contrib/lizard/lib/lizard_frame.c create mode 100644 contrib/lizard/lib/lizard_frame.h create mode 100644 contrib/lizard/lib/lizard_frame_static.h create mode 100644 contrib/lizard/lib/lizard_parser_fast.h create mode 100644 contrib/lizard/lib/lizard_parser_fastbig.h create mode 100644 contrib/lizard/lib/lizard_parser_fastsmall.h create mode 100644 contrib/lizard/lib/lizard_parser_hashchain.h create mode 100644 contrib/lizard/lib/lizard_parser_lowestprice.h create mode 100644 contrib/lizard/lib/lizard_parser_nochain.h create mode 100644 contrib/lizard/lib/lizard_parser_optimal.h create mode 100644 contrib/lizard/lib/lizard_parser_pricefast.h create mode 100644 contrib/lizard/lib/xxhash/xxhash.c create mode 100644 contrib/lizard/lib/xxhash/xxhash.h create mode 100644 contrib/lizard/programs/.gitignore create mode 100644 contrib/lizard/programs/COPYING create mode 100644 contrib/lizard/programs/README.md create mode 100644 contrib/lizard/programs/bench.c create mode 100644 contrib/lizard/programs/bench.h create mode 100644 contrib/lizard/programs/datagen.c create mode 100644 contrib/lizard/programs/datagen.h create mode 100644 contrib/lizard/programs/lizard.1 create mode 100644 contrib/lizard/programs/lizardcli.c create mode 100644 contrib/lizard/programs/lizardio.c create mode 100644 contrib/lizard/programs/lizardio.h create mode 100644 contrib/lizard/programs/platform.h create mode 100644 contrib/lizard/programs/util.h create mode 100644 contrib/lizard/tests/.gitignore create mode 100644 contrib/lizard/tests/COPYING create mode 100644 contrib/lizard/tests/README.md create mode 100644 contrib/lizard/tests/datagencli.c create mode 100644 contrib/lizard/tests/frametest.c create mode 100644 contrib/lizard/tests/fullbench.c create mode 100644 contrib/lizard/tests/fuzzer.c create mode 100644 contrib/lizard/visual/.gitignore create mode 100644 contrib/lizard/visual/README.md create mode 100644 contrib/lizard/visual/VS2010/datagen/datagen.vcxproj create mode 100644 contrib/lizard/visual/VS2010/frametest/frametest.vcxproj create mode 100644 contrib/lizard/visual/VS2010/fullbench/fullbench.vcxproj create mode 100644 contrib/lizard/visual/VS2010/fuzzer/fuzzer.vcxproj create mode 100644 contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.rc create mode 100644 contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.vcxproj create mode 100644 contrib/lizard/visual/VS2010/liblizard/liblizard.vcxproj create mode 100644 contrib/lizard/visual/VS2010/lizard.sln create mode 100644 contrib/lizard/visual/VS2010/lizard/lizard.rc create mode 100644 contrib/lizard/visual/VS2010/lizard/lizard.vcxproj create mode 100644 contrib/lzsse/.gitignore create mode 100644 contrib/lzsse/.travis.yml create mode 100644 contrib/lzsse/LICENSE create mode 100644 contrib/lzsse/README.md create mode 100644 contrib/lzsse/example/main.cpp create mode 100644 contrib/lzsse/lzsse2/lzsse2.cpp create mode 100644 contrib/lzsse/lzsse2/lzsse2.h create mode 100644 contrib/lzsse/lzsse2/lzsse2_platform.h create mode 100644 contrib/lzsse/lzsse4/lzsse4.cpp create mode 100644 contrib/lzsse/lzsse4/lzsse4.h create mode 100644 contrib/lzsse/lzsse4/lzsse4_platform.h create mode 100644 contrib/lzsse/lzsse8/lzsse8.cpp create mode 100644 contrib/lzsse/lzsse8/lzsse8.h create mode 100644 contrib/lzsse/lzsse8/lzsse8_platform.h create mode 100644 contrib/lzsse/premake4.exe create mode 100644 contrib/lzsse/premake4.lua diff --git a/contrib/density/.gitignore b/contrib/density/.gitignore new file mode 100644 index 00000000000..5192545dd3f --- /dev/null +++ b/contrib/density/.gitignore @@ -0,0 +1,2 @@ +build/ +benchmark/build diff --git a/contrib/density/.gitmodules b/contrib/density/.gitmodules new file mode 100644 index 00000000000..91eee619841 --- /dev/null +++ b/contrib/density/.gitmodules @@ -0,0 +1,6 @@ +[submodule "benchmark/libs/cputime"] + path = benchmark/libs/cputime + url = https://github.com/k0dai/cputime.git +[submodule "benchmark/libs/spookyhash"] + path = benchmark/libs/spookyhash + url = https://github.com/k0dai/spookyhash.git diff --git a/contrib/density/.travis.yml b/contrib/density/.travis.yml new file mode 100644 index 00000000000..62e7655a4cb --- /dev/null +++ b/contrib/density/.travis.yml @@ -0,0 +1,272 @@ +language: c +matrix: + fast_finish: true + include: + - os: osx + env: + - C_COMPILER=clang + + - os: osx + env: + - C_COMPILER=gcc + + - dist: trusty + sudo: required + addons: + apt: + packages: + - qemu-system-arm + - qemu-user-static + - gcc-aarch64-linux-gnu + - g++-aarch64-linux-gnu + env: + - C_COMPILER=aarch64-linux-gnu-gcc MAKE_OPTIONS="AR=gcc-ar NATIVE=off" QEMU_INTERPRETER=qemu-aarch64-static PRE_SCRIPT="export QEMU_LD_PREFIX=/usr/aarch64-linux-gnu/" + + - dist: trusty + sudo: required + addons: + apt: + packages: + - qemu-system-arm + - qemu-user-static + - gcc-arm-linux-gnueabi + - g++-arm-linux-gnueabi + env: + - C_COMPILER=arm-linux-gnueabi-gcc MAKE_OPTIONS="AR=gcc-ar NATIVE=off" QEMU_INTERPRETER=qemu-arm-static PRE_SCRIPT="export QEMU_LD_PREFIX=/usr/arm-linux-gnueabi/" + + - dist: trusty + sudo: required + addons: + apt: + packages: + - qemu-system-ppc + - qemu-user-static + - gcc-powerpc-linux-gnu + - g++-powerpc-linux-gnu + env: + - C_COMPILER=powerpc-linux-gnu-gcc MAKE_OPTIONS="AR=gcc-ar ARCH=64" QEMU_INTERPRETER="qemu-ppc64-static" PRE_SCRIPT="export QEMU_LD_PREFIX=/usr/powerpc-linux-gnu/" + + - dist: trusty + sudo: required + addons: + apt: + packages: + - qemu-system-ppc + - qemu-user-static + - gcc-powerpc-linux-gnu + - g++-powerpc-linux-gnu + env: + - C_COMPILER=powerpc-linux-gnu-gcc MAKE_OPTIONS="AR=gcc-ar NATIVE=off" QEMU_INTERPRETER="qemu-ppc-static" PRE_SCRIPT="export QEMU_LD_PREFIX=/usr/powerpc-linux-gnu/" + + - dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.8 + env: + - C_COMPILER=gcc-4.8 MAKE_OPTIONS="AR=gcc-ar" + + - dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.8 + - gcc-4.8-multilib + env: + - C_COMPILER=gcc-4.8 MAKE_OPTIONS="AR=gcc-ar ARCH=32" + + - dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.9 + env: + - C_COMPILER=gcc-4.9 MAKE_OPTIONS="AR=gcc-ar" + + - dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.9 + - gcc-4.9-multilib + env: + - C_COMPILER=gcc-4.9 MAKE_OPTIONS="AR=gcc-ar ARCH=32" + + - dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-5 + env: + - C_COMPILER=gcc-5 MAKE_OPTIONS="AR=gcc-ar-5" + + - dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-5 + - gcc-5-multilib + env: + - C_COMPILER=gcc-5 MAKE_OPTIONS="AR=gcc-ar-5 ARCH=32" + +# - dist: trusty +# sudo: required +# addons: +# apt: +# sources: +# - ubuntu-toolchain-r-test +# packages: +# - gcc-6 +# env: +# - C_COMPILER=gcc-6 MAKE_OPTIONS="AR=gcc-ar-6" + + - dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-6 + - gcc-6-multilib + env: + - C_COMPILER=gcc-6 MAKE_OPTIONS="AR=gcc-ar-6 ARCH=32" + +# - dist: trusty +# sudo: required +# addons: +# apt: +# sources: +# - ubuntu-toolchain-r-test +# packages: +# - gcc-7 +# env: +# - C_COMPILER=gcc-7 MAKE_OPTIONS="AR=gcc-ar-7" + + - dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-7 + - gcc-7-multilib + env: + - C_COMPILER=gcc-7 MAKE_OPTIONS="AR=gcc-ar-7 ARCH=32" + + - dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-trusty-3.5 + packages: + - clang-3.5 + - llvm-3.5-dev + env: + - C_COMPILER=clang-3.5 MAKE_OPTIONS="AR=llvm-ar-3.5" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" + + - dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-trusty-3.6 + packages: + - clang-3.6 + - llvm-3.6-dev + env: + - C_COMPILER=clang-3.6 MAKE_OPTIONS="AR=llvm-ar-3.6" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" + +# - dist: precise +# sudo: required +# addons: +# apt: +# sources: +# - ubuntu-toolchain-r-test +# - llvm-toolchain-precise-3.7 +# packages: +# - clang-3.7 +# - llvm-3.7-dev +# env: +# - C_COMPILER=clang-3.7 MAKE_OPTIONS="AR=llvm-ar-3.7" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" + +# - dist: precise +# sudo: required +# addons: +# apt: +# sources: +# - ubuntu-toolchain-r-test +# - llvm-toolchain-precise-3.8 +# packages: +# - clang-3.8 +# - llvm-3.8-dev +# env: +# - C_COMPILER=clang-3.8 MAKE_OPTIONS="AR=llvm-ar-3.8" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" + + - dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-trusty-3.9 + packages: + - clang-3.9 + - llvm-3.9-dev + env: + - C_COMPILER=clang-3.9 MAKE_OPTIONS="AR=llvm-ar-3.9" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" + + - dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-trusty-4.0 + packages: + - clang-4.0 + - llvm-4.0-dev + env: + - C_COMPILER=clang-4.0 MAKE_OPTIONS="AR=llvm-ar-4.0" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" + +# - dist: trusty +# sudo: required +# addons: +# apt: +# sources: +# - ubuntu-toolchain-r-test +# - llvm-toolchain-trusty-5.0 +# packages: +# - clang-5.0 +# - llvm-5.0-dev +# env: +# - C_COMPILER=clang-5.0 MAKE_OPTIONS="AR=llvm-ar-5.0" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" + +script: + - ${PRE_SCRIPT} + - $C_COMPILER -v + - make CC=$C_COMPILER $MAKE_OPTIONS + - file ./build/* + - $QEMU_INTERPRETER ./build/benchmark -h -f diff --git a/contrib/density/CHANGELOG.md b/contrib/density/CHANGELOG.md new file mode 100644 index 00000000000..6d5535bcc7c --- /dev/null +++ b/contrib/density/CHANGELOG.md @@ -0,0 +1,168 @@ +0.14.2 +------ +*February 12, 2018* +* Improved chameleon decode speed +* Added data hash checks and display option in benchmark +* Now using makefiles as build system +* Big endian support correctly implemented and tested +* Improved continuous integration tests + +0.14.1 +------ +*January 20, 2018* +* Added MSVC support +* Added continuous integration on travis and appveyor +* Premake script improvement +* Various codebase improvements + +0.14.0 +------ +*January 16, 2018* +* First stable version of DENSITY +* Complete project reorganization and API rewrite +* Many stability fixes and improvements +* Fast revert to conditional copy for incompressible input +* Custom dictionaries in API +* Improvements in compression ratio and speed + +0.12.5 beta +----------- +*June 20, 2015* + +* Added conditional main footer read/write +* Improved teleport staging buffer management +* Regression - a minimum buffer output size has to be ensured to avoid signature loss +* Modified the minimum lookahead and the resulting minimum buffer size in the API +* Lion : corrected a signature interception problem due to an increase in process unit size +* Lion : corrected chunk count conditions for new block / mode marker detection +* Lion : modified end of stream marker conditions +* Stability fixes and improvements + +0.12.4 beta +----------- +*May 25, 2015* + +* Removed remaining undefined behavior potential occurences +* Implemented parallelizable decompressible output block header reads/writes (disabled by default) + +0.12.3 beta +----------- +*May 20, 2015* + +* New lion algorithm, faster and more efficient +* Compiler specific optimizations +* Switched to premake 5 to benefit from link time optimizations +* Various fixes and improvements + +0.12.2 beta +----------- +*May 4, 2015* + +* Added an integrated in-memory benchmark +* Better Windows compatibility +* Fixed misaligned load/stores +* Switched to the premake build system +* Performance optimizations (pointers, branches, loops ...) +* Various fixes and improvements + +0.12.1 beta +----------- +*April 3, 2015* + +* Better unrolling readability and efficiency +* Improved read speed of dictionary/predictions entries +* Implemented case generators in cheetah to speed up decoding by using less branches +* Added signatures interception in lion to cancel the need for large output buffers +* Improved lion decode speed with specific form data access and use of ctz in form read +* Enabled decompression to exact-sized buffer for all algorithms +* Various fixes and improvements + +0.12.0 beta +----------- +*March 24, 2015* + +* Added new lion kernel +* Renamed kernel mandala to cheetah +* Kernel chameleon and cheetah improvements in encoding/decoding speeds +* Generic function macros to avoid code rewrite +* Improved memory teleport IO flexibility and speed, bytes issued by memory teleport can now be partially read +* Various fixes and improvements + +0.11.3 beta +----------- +*February 5, 2015* + +* Added integrity check system +* Corrected pointer usage and update on footer read/writes +* Now freeing kernel state memory only when compression mode is not copy +* Updated Makefiles +* Improved memory teleport +* Fixed sequencing problem after kernels request a new block + +0.11.2 beta +----------- +*February 3, 2015* + +* Added an algorithms overview in README +* Removed ssc references +* Now initializing last hash to zero on mandala kernel inits +* Reimplemented the buffer API +* Various corrections and improvements + +0.11.1 beta +----------- +*January 19, 2015* + +* Added a sharc benchmark in README +* Stateless memory teleport +* Improved event management and dispatching +* Improved compression/decompression finishes +* Improved streams API +* Various bug fixes, robustness improvements + +0.10.2 beta +----------- +*January 7, 2015* + +* Improved organization of compile-time switches and run-time options in the API +* Removed method density_stream_decompress_utilities_get_header from the API, header info is now returned in the density_stream_decompress_init function +* Corrected readme to reflect API changes + +0.10.1 beta +----------- +*January 5, 2015* + +* Re-added mandala kernel +* Corrected available bytes adjustment problem +* Added missing restrict keywords +* Cleaned unnecessary defines + +0.10.0 beta +----------- +*January 2, 2015* + +* Complete stream API redesign to greatly improve flexibility +* Only one supported algorithm for now : Chameleon + +0.9.12 beta +----------- +*December 2, 2013* + +* Mandala kernel addition, replacing dual pass chameleon +* Simplified, faster hash function +* Fixed memory freeing issue during main encoding/decoding finish +* Implemented no footer encode output type +* Namespace migration, kernel structure reorganization +* Corrected copy mode problem +* Implemented efficiency checks and mode reversions +* Corrected lack of main header parameters retrieval +* Fixed stream not being properly ended when mode reversion occurred +* Updated metadata computations + +0.9.11 beta +----------- +*November 2, 2013* + +* First beta release of DENSITY, including all the compression code from SHARC in a standalone, BSD licensed library +* Added copy mode (useful for enhancing data security via the density block checksums for example) +* Makefile produces static and dynamic libraries diff --git a/contrib/density/LICENSE.md b/contrib/density/LICENSE.md new file mode 100644 index 00000000000..c2b699c9b83 --- /dev/null +++ b/contrib/density/LICENSE.md @@ -0,0 +1,28 @@ +Copyright (c) 2013, Guillaume Voirin + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/density/README.md b/contrib/density/README.md new file mode 100644 index 00000000000..6c660dea1f5 --- /dev/null +++ b/contrib/density/README.md @@ -0,0 +1,261 @@ +DENSITY +======== +Superfast compression library + +DENSITY is a free C99, open-source, BSD licensed compression library. + +It is focused on high-speed compression, at the best ratio possible. **All three** of DENSITY's algorithms are currently at the **pareto frontier** of compression speed vs ratio (cf. [here](https://github.com/inikep/lzbench/blob/master/lzbench18_sorted.md) for an independent benchmark). + +DENSITY features a simple API to enable quick integration in any project. + +Branch|Linux & MacOS|Windows +--- | --- | --- +master|[![Build Status](https://travis-ci.org/k0dai/density.svg?branch=master)](https://travis-ci.org/k0dai/density)|[![Build status](https://ci.appveyor.com/api/projects/status/rf7x3x829il72cii/branch/master?svg=true)](https://ci.appveyor.com/project/gpnuma/density/branch/master) +dev|[![Build Status](https://travis-ci.org/k0dai/density.svg?branch=dev)](https://travis-ci.org/k0dai/density)|[![Build status](https://ci.appveyor.com/api/projects/status/rf7x3x829il72cii/branch/dev?svg=true)](https://ci.appveyor.com/project/gpnuma/density/branch/dev) + +Why is it so fast ? +------------------- + +One of the biggest assets of DENSITY is that its work unit is **not a byte** like other libraries, but **a group of 4 bytes**. + +When other libraries consume one byte of data and then apply an algorithmic processing to it, DENSITY consumes 4 bytes and then applies its algorithmic processing. + +That's why DENSITY's algorithms were designed from scratch. They have to alleviate for 4-byte work units and still provide interesting compression ratios. + +**Speed pedigree traits** + +* 4-byte work units +* heavy use of registers as opposed to memory for processing +* avoidance of or use of minimal branching when possible +* use of low memory data structures to favor processor cache Lx accesses +* library wide inlining +* specific unrollings +* prefetching and branching hints +* restricted pointers to maximize compiler optimizations + +A "blowup protection" is provided, dramatically increasing the processing speed of incompressible input data. Also, the output, compressed data size will **never exceed** the original uncompressed data size by more than 1% in case of incompressible, reasonably-sized inputs. + +Benchmarks +---------- + +**Quick benchmark** + +DENSITY features an **integrated in-memory benchmark**. After building the project (see [build](#build)), a *benchmark* executable will be present in the build directory. If run without arguments, usage help will be displayed. + +File used : enwik8 (100 MB) + +Platform : MacBook Pro, MacOS 10.13.3, 2.3 GHz Intel Core i7, 8Gb 1600 MHz DDR, SSD, compiling with Clang/LLVM 9.0.0 + +Timing : using the *time* function, and taking the best *user* output after multiple runs. In the case of density, the in-memory integrated benchmark's best value (which uses the same usermode CPU timing) is used. + +Library|Algorithm|Compress|Decompress|Size|Ratio|Round trip +---|---|---|---|---|---|--- +**density** 0.14.2|Chameleon|0.092s (1085 MB/s)|0.059s (1684 MB/s)|61 524 084|61,52%|0.151s +lz4 r129|-1|0.468s (214 MB/s)|0.115s (870 MB/s)|57 285 990|57,29%|0.583s +lzo 2.08|-1|0.367s (272 MB/s)|0.309s (324 MB/s)|56 709 096|56,71%|0.676s +**density** 0.14.2|Cheetah|0.170s (587 MB/s)|0.126s (796 MB/s)|53 156 668|53,16%|0.296s +**density** 0.14.2|Lion|0.303s (330 MB/s)|0.288s (347 MB/s)|47 817 692|47,82%|0.591s +lz4 r129|-3|1.685s (59 MB/s)|0.118s (847 MB/s)|44 539 940|44,54%|1.803s +lzo 2.08|-7|9.562s (10 MB/s)|0.319s (313 MB/s)|41 720 721|41,72%|9.881s + +**Other benchmarks** + +Here are a few other benchmarks featuring DENSITY (non exhaustive list) : + +* [**squash**](https://github.com/quixdb/squash) is an abstraction layer for compression algorithms, and has an extremely exhaustive set of benchmark results, including density's, [available here](https://quixdb.github.io/squash-benchmark/?dataset=dickens&machine=s-desktop). + +* [**lzbench**](https://github.com/inikep/lzbench) is an in-memory benchmark of open-source LZ77/LZSS/LZMA compressors. + +* [**fsbench**](https://github.com/gpnuma/fsbench-density) is a command line utility that enables real-time testing of compression algorithms, but also hashes and much more. A fork with density releases is [available here](https://github.com/gpnuma/fsbench-density) for easy access. +The original author's repository [can be found here](https://chiselapp.com/user/Justin_be_my_guide/repository/fsbench/). + +Build +----- +DENSITY can be built on a number of platforms, via the provided makefiles. + +It was developed and optimized against Clang/LLVM which makes it the preferred compiler, but GCC and MSVC are also supported. Please use the latest compiler versions for best performance. + +**MacOS** + +On MacOS, Clang/LLVM is the default compiler, which makes things simpler. + +1) Get the source code : + +``` + git clone https://github.com/k0dai/density.git + cd density +``` + +2) Build and test : + +``` + make + build/benchmark -f +``` + +Alternatively, thanks to the [Homebrew project](https://brew.sh), DENSITY can also be installed with a single command on MacOS: + +``` + brew install density +``` + +**Linux** + +On Linux, Clang/LLVM is not always available by default, but can be easily added thanks to the provided package managers. +The following example assumes a Debian or Ubuntu distribution with *apt-get*. + +1) From the command line, install Clang/LLVM (*optional*, GCC is also supported if Clang/LLVM can't be used) and other prerequisites. + +``` + sudo apt-get install clang git +``` + +2) Get the source code : + +``` + git clone https://github.com/k0dai/density.git + cd density +``` + +3) Build and test : + +``` + make +``` +or +``` + make CC=gcc-... AR=gcc-ar-... +``` +or +``` + make CC=clang-... AR=llvm-ar-... +``` +to choose alternative compilers. For a quick test of resulting binaries, run +``` + build/benchmark -f +``` + +**Windows** + +Please install [git for Windows](https://git-scm.com/download/win) to begin with. + +On Windows, density can be built in different ways. +The **first method** is to use mingw's gcc compiler; for that it is necessary to download and install [mingw-w64](https://sourceforge.net/projects/mingw-w64/). + +1) Once mingw-w64 is installed, get the source : + +``` + git clone https://github.com/k0dai/density.git + cd density +``` + +2) Build and test : + +``` + mingw32-make.exe + build/benchmark.exe -f +``` + +As an alternative, [MSYS2](http://www.msys2.org/) also offers a linux-like environment for Windows. + +The **second method** is to download and install Microsoft's [Visual Studio IDE community edition](https://www.visualstudio.com/thank-you-downloading-visual-studio/?sku=Community). It comes with Microsoft's own compilers and is free. + +1) Once Visual Studio is installed, open a [developer command prompt](https://docs.microsoft.com/en-us/dotnet/framework/tools/developer-command-prompt-for-vs) and type : + +``` + git clone https://github.com/k0dai/density.git + cd density\msvc +``` + +2) Build and test : + +``` + msbuild Density.sln + bin\Release\benchmark.exe -f +``` + +An extra **recommended step** would be to install *Clang/LLVM* for Windows. It is downloadable from [this link](http://releases.llvm.org/5.0.1/LLVM-5.0.1-win64.exe). Once installed, open the Visual Studio IDE by double-clicking on *Density.sln*, then right-click on project names and change the platform toolsets to *LLVM*. Rebuild the solution to generate binaries with Clang/LLVM. + +Output format +------------- +DENSITY outputs compressed data in a simple format, which enables file storage and optional parallelization for both compression and decompression. + +A very short header holding vital informations (like DENSITY version and algorithm used) precedes the binary compressed data. + +APIs +---- +DENSITY features a straightforward *API*, simple yet powerful enough to keep users' creativity unleashed. + +For advanced developers, it allows use of custom dictionaries and exportation of generated dictionaries after a compression session. Although using the default, blank dictionary is perfectly fine in most cases, setting up your own, tailored dictionaries could somewhat improve compression ratio especially for low sized input datum. + +Please see the [*quick start*](#quick-start-a-simple-example-using-the-api) at the bottom of this page. + +About the algorithms +-------------------- + +**Chameleon** ( *DENSITY_ALGORITHM_CHAMELEON* ) + +Chameleon is a dictionary lookup based compression algorithm. It is designed for absolute speed and usually reaches a 60% compression ratio on compressible data. +Decompression is just as fast. This algorithm is a great choice when main concern is speed. + +**Cheetah** ( *DENSITY_ALGORITHM_CHEETAH* ) + +Cheetah was developed with inputs from [Piotr Tarsa](https://github.com/tarsa). +It is derived from chameleon and uses swapped double dictionary lookups and predictions. It can be extremely good with highly compressible data (ratio reaching 10% or less). +On typical compressible data compression ratio is about 50% or less. It is still extremely fast for both compression and decompression and is a great, efficient all-rounder algorithm. + +**Lion** ( *DENSITY_ALGORITHM_LION* ) + +Lion is a multiform compression algorithm derived from cheetah. It goes further in the areas of dynamic adaptation and fine-grained analysis. +It uses multiple swapped dictionary lookups and predictions, and forms rank entropy coding. +Lion provides the best compression ratio of all three algorithms under any circumstance, and is still very fast. + +Quick start (a simple example using the API) +-------------------------------------------- +Using DENSITY in your application couldn't be any simpler. + +First you need to include this file in your project : + +* density_api.h + +When this is done you can start using the **DENSITY API** : + +```C + #include + #include "density_api.h" + + char* text = "This is a simple example on how to use the simple Density API. This is a simple example on how to use the simple Density API."; + uint64_t text_length = (uint64_t)strlen(text); + + // Determine safe buffer sizes + uint_fast64_t compress_safe_size = density_compress_safe_size(text_length); + uint_fast64_t decompress_safe_size = density_decompress_safe_size(text_length); + + // Allocate required memory + uint8_t *outCompressed = malloc(compress_safe_size * sizeof(char)); + uint8_t *outDecompressed = malloc(decompress_safe_size * sizeof(char)); + density_processing_result result; + + // Compress + result = density_compress(text, text_length, outCompressed, compress_safe_size, DENSITY_COMPRESSION_MODE_CHAMELEON_ALGORITHM); + if(!result.state) + printf("Compressed %llu bytes to %llu bytes\n", result.bytesRead, result.bytesWritten); + + // Decompress + result = density_decompress(outCompressed, result.bytesWritten, outDecompressed, decompress_safe_size); + if(!result.state) + printf("Decompressed %llu bytes to %llu bytes\n", result.bytesRead, result.bytesWritten); + + // Free memory_allocated + free(outCompressed); + free(outDecompressed); +``` + +And that's it ! We've done a compression/decompression round trip with a few lines ! + +Related projects +---------------- + +* **SHARC** (archiver using density algorithms) [https://github.com/gpnuma/sharc](https://github.com/gpnuma/sharc) +* **fsbench-density** (in-memory transformations benchmark) [https://github.com/gpnuma/fsbench-density](https://github.com/gpnuma/fsbench-density) +* **densityxx** (c++ port of density) [https://github.com/charlesw1234/densityxx](https://github.com/charlesw1234/densityxx) diff --git a/contrib/density/appveyor.yml b/contrib/density/appveyor.yml new file mode 100644 index 00000000000..0aabedebc8b --- /dev/null +++ b/contrib/density/appveyor.yml @@ -0,0 +1,37 @@ +version: 0.14.2.{build} +image: Visual Studio 2015 +init: +- cmd: +environment: + matrix: + - EXTRA_PATH: C:\mingw-w64\i686-5.3.0-posix-dwarf-rt_v4-rev0\mingw32\bin\ + LAUNCH_BUILD: mingw32-make CC=gcc AR=gcc-ar + LAUNCH_PATH: build\ + - EXTRA_PATH: C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\ + LAUNCH_BUILD: mingw32-make CC=gcc AR=gcc-ar + LAUNCH_PATH: build\ + - EXTRA_PATH: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\ + LAUNCH_BUILD: mingw32-make CC=gcc AR=gcc-ar + LAUNCH_PATH: build\ + - EXTRA_PATH: C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin\ + LAUNCH_BUILD: mingw32-make CC=gcc AR=gcc-ar + LAUNCH_PATH: build\ + - EXTRA_PATH: '%programfiles(x86)%\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\amd64\' + LAUNCH_BUILD: cd msvc && msbuild.exe Density.sln + EXTRA_GIT: git submodule update --init --recursive + LAUNCH_PATH: bin\Release\ +install: +- cmd: +build_script: +- cmd: >- + %EXTRA_GIT% + + set PATH=%EXTRA_PATH%;%PATH% + + %LAUNCH_BUILD% + + file %LAUNCH_PATH:\=/%* + + %LAUNCH_PATH%benchmark.exe -h -f +test_script: +- cmd: diff --git a/contrib/density/benchmark/src/benchmark.c b/contrib/density/benchmark/src/benchmark.c new file mode 100644 index 00000000000..9781ef361c6 --- /dev/null +++ b/contrib/density/benchmark/src/benchmark.c @@ -0,0 +1,345 @@ +/* + * Density benchmark + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * 5/04/15 19:30 + */ + +#include "benchmark.h" + +void density_benchmark_version() { + printf("\nSingle threaded "); + DENSITY_BENCHMARK_BOLD(printf("in-memory benchmark")); + printf(" powered by "); + DENSITY_BENCHMARK_BOLD(printf("Density %i.%i.%i\n", density_version_major(), density_version_minor(), density_version_revision())); + printf("Copyright (C) 2015 Guillaume Voirin\n"); + printf("Built for %s (%s endian system, %u bits) using " DENSITY_BENCHMARK_COMPILER ", %s %s\n", DENSITY_BENCHMARK_PLATFORM_STRING, DENSITY_BENCHMARK_ENDIAN_STRING, (unsigned int) (8 * sizeof(void *)), DENSITY_BENCHMARK_COMPILER_VERSION, __DATE__, __TIME__); +} + +void density_benchmark_client_usage() { + printf("\n"); + DENSITY_BENCHMARK_BOLD(printf("Usage :\n")); + printf(" benchmark [OPTIONS ?]... [FILE ?]\n\n"); + DENSITY_BENCHMARK_BOLD(printf("Available options :\n")); + printf(" -[LEVEL] Test file using only the specified compression LEVEL\n"); + printf(" If unspecified, all algorithms are tested (default).\n"); + printf(" LEVEL can have the following values (as values become higher,\n"); + printf(" compression ratio increases and speed diminishes) :\n"); + printf(" 0 = Copy (no compression)\n"); + printf(" 1 = Chameleon algorithm\n"); + printf(" 2 = Cheetah algorithm\n"); + printf(" 3 = Lion algorithm\n"); + printf(" -c Compress only\n"); + printf(" -f Activate fuzzer mode (pseudorandom generated data)\n"); + printf(" -h Print data hashing informations\n\n"); + exit(EXIT_SUCCESS); +} + +void density_benchmark_format_decimal(uint64_t number) { + if (number < 1000) { + printf("%"PRIu64, number); + return; + } + density_benchmark_format_decimal(number / 1000); + printf(",%03"PRIu64, number % 1000); +} + +const char *density_benchmark_convert_state_to_text(DENSITY_STATE state) { + switch (state) { + case DENSITY_STATE_ERROR_DURING_PROCESSING: + return "Error during processing"; + case DENSITY_STATE_ERROR_INPUT_BUFFER_TOO_SMALL: + return "Input buffer is too small"; + case DENSITY_STATE_ERROR_OUTPUT_BUFFER_TOO_SMALL: + return "Output buffer is too small"; + case DENSITY_STATE_ERROR_INVALID_CONTEXT: + return "Invalid context"; + case DENSITY_STATE_ERROR_INVALID_ALGORITHM: + return "Invalid algorithm"; + default: + return "Unknown error"; + } +} + +int main(int argc, char *argv[]) { + density_benchmark_version(); + DENSITY_ALGORITHM start_mode = DENSITY_ALGORITHM_CHAMELEON; + DENSITY_ALGORITHM end_mode = DENSITY_ALGORITHM_LION; + bool compression_only = false; + bool fuzzer = false; + bool hash_info = false; + char *file_path = NULL; + + if (argc <= 1) + density_benchmark_client_usage(); + for (int count = 1; count < argc; count++) { + if (argv[count][0] == '-') { + switch (argv[count][1]) { + case '1': + start_mode = DENSITY_ALGORITHM_CHAMELEON; + end_mode = DENSITY_ALGORITHM_CHAMELEON; + break; + case '2': + start_mode = DENSITY_ALGORITHM_CHEETAH; + end_mode = DENSITY_ALGORITHM_CHEETAH; + break; + case '3': + start_mode = DENSITY_ALGORITHM_LION; + end_mode = DENSITY_ALGORITHM_LION; + break; + case 'c': + compression_only = true; + break; + case 'f': + fuzzer = true; + break; + case 'h': + hash_info = true; + break; + default: + density_benchmark_client_usage(); + } + } else + file_path = argv[argc - 1]; + } + + uint8_t *in; + uint8_t *out; + uint_fast64_t uncompressed_size; + uint_fast64_t memory_allocated; + if (fuzzer) { + srand((unsigned int) (time(NULL) * 14521937821257379531llu)); + uncompressed_size = (uint_fast64_t) (((uint64_t) (rand() * 100000000llu)) / RAND_MAX); + memory_allocated = density_compress_safe_size(uncompressed_size); + in = malloc(memory_allocated * sizeof(uint8_t)); + uint8_t value = (uint8_t) rand(); + for (unsigned int count = 0; count < uncompressed_size; count++) { + if (!(rand() & 0xf)) + value += (uint8_t)rand(); + in[count] = value; + } + out = malloc(memory_allocated * sizeof(uint8_t)); + } else { + // Open file and get infos + FILE *file = fopen(file_path, "rb"); + if (file == NULL) { + DENSITY_BENCHMARK_ERROR(printf("Error opening file %s.", file_path), false); + } + struct stat file_attributes; + stat(file_path, &file_attributes); + + // Allocate memory and copy file to memory + uncompressed_size = (uint_fast64_t) file_attributes.st_size; + memory_allocated = density_compress_safe_size(uncompressed_size); + in = malloc(memory_allocated * sizeof(uint8_t)); + size_t read = fread(in, sizeof(uint8_t), uncompressed_size, file); + if(uncompressed_size != read) { + DENSITY_BENCHMARK_ERROR(printf("Error reading file %s.", file_path), false); + } + fclose(file); + out = malloc(memory_allocated * sizeof(uint8_t)); + } + + printf("Allocated "); + density_benchmark_format_decimal(2 * memory_allocated); + printf(" bytes of in-memory work space\n"); + + uint64_t original_hash_1 = DENSITY_BENCHMARK_HASH_SEED_1; + uint64_t original_hash_2 = DENSITY_BENCHMARK_HASH_SEED_2; + spookyhash_128(in, uncompressed_size, &original_hash_1, &original_hash_2); + + printf("\n"); + for (DENSITY_ALGORITHM compression_mode = start_mode; compression_mode <= end_mode; compression_mode++) { + // Print algorithm info + switch (compression_mode) { + case DENSITY_ALGORITHM_CHAMELEON: + DENSITY_BENCHMARK_BLUE(DENSITY_BENCHMARK_BOLD(printf("Chameleon algorithm"))); + DENSITY_BENCHMARK_UNDERLINE(19); + break; + case DENSITY_ALGORITHM_CHEETAH: + DENSITY_BENCHMARK_BLUE(DENSITY_BENCHMARK_BOLD(printf("Cheetah algorithm"))); + DENSITY_BENCHMARK_UNDERLINE(17); + break; + case DENSITY_ALGORITHM_LION: + DENSITY_BENCHMARK_BLUE(DENSITY_BENCHMARK_BOLD(printf("Lion algorithm"))); + DENSITY_BENCHMARK_UNDERLINE(14); + break; + } + fflush(stdout); + + // Pre-heat + printf("\nUsing "); + if (fuzzer) { + DENSITY_BENCHMARK_BOLD(printf("generated data")); + } else { + printf("file "); + DENSITY_BENCHMARK_BOLD(printf("%s", file_path)); + } + printf(" copied in memory\n"); + if(hash_info) { + printf("Uncompressed data hash is "); + DENSITY_BENCHMARK_BOLD(printf("0x%" PRIx64 "%" PRIx64, original_hash_1, original_hash_2)); + printf("\n"); + } + + density_processing_result result = density_compress(in, uncompressed_size, out, memory_allocated, compression_mode); + if (result.state) { + DENSITY_BENCHMARK_ERROR(printf("During compress API returned error %i (%s).", result.state, density_benchmark_convert_state_to_text(result.state)), true); + } + const uint_fast64_t compressed_size = result.bytesWritten; + + uint64_t hash_1 = DENSITY_BENCHMARK_HASH_SEED_1; + uint64_t hash_2 = DENSITY_BENCHMARK_HASH_SEED_2; + if(hash_info) { + spookyhash_128(out, compressed_size, &hash_1, &hash_2); + printf("Compressed data hash is "); + DENSITY_BENCHMARK_BOLD(printf("0x%" PRIx64 "%" PRIx64, hash_1, hash_2)); + printf("\n"); + } + + if (!compression_only) { + memset(in, 0, memory_allocated); + result = density_decompress(out, compressed_size, in, memory_allocated); + if (result.state) { + DENSITY_BENCHMARK_ERROR(printf("During decompress API returned error %i (%s).", result.state, density_benchmark_convert_state_to_text(result.state)), true); + } + if (result.bytesWritten != uncompressed_size) { + DENSITY_BENCHMARK_ERROR(printf("Round-trip size differs from original size ("); + density_benchmark_format_decimal(result.bytesWritten); + printf(" bytes against "); + density_benchmark_format_decimal(uncompressed_size); + printf(" bytes).");, true); + } + + hash_1 = DENSITY_BENCHMARK_HASH_SEED_1; + hash_2 = DENSITY_BENCHMARK_HASH_SEED_2; + spookyhash_128(in, uncompressed_size, &hash_1, &hash_2); + + if(hash_info) { + printf("Round-trip data hash is "); + DENSITY_BENCHMARK_BOLD(printf("0x%" PRIx64 "%" PRIx64, hash_1, hash_2)); + printf("\n"); + } + + if(hash_1 != original_hash_1 || hash_2 != original_hash_2) { + DENSITY_BENCHMARK_ERROR(printf("Uncompressed and round-trip data hashes do not match ("); + printf("0x%" PRIx64 "%" PRIx64, hash_1, hash_2); + printf(" vs. "); + printf("0x%" PRIx64 "%" PRIx64, original_hash_1, original_hash_2); + printf(").");, true); + } + + printf("Uncompressed and round-trip data hashes match. "); + } + printf("Starting main bench.\n"); + if (compression_only) + printf("Compressing "); + else + printf("Round-tripping "); + density_benchmark_format_decimal(uncompressed_size); + printf(" bytes to "); + density_benchmark_format_decimal(compressed_size); + printf(" bytes (compression ratio "); + DENSITY_BENCHMARK_BOLD(printf("%.2lf%%", (100.0 * compressed_size) / uncompressed_size)); + printf(" or "); + DENSITY_BENCHMARK_BOLD(printf("%.3fx", (1.0 * uncompressed_size) / compressed_size)); + if (compression_only) + printf(")\n"); + else + printf(") and back\n"); + fflush(stdout); + + // Main benchmark + unsigned int iterations = 0; + double compress_time_high = 0.0; + double compress_time_low = 60.0; + double decompress_time_high = 0.0; + double decompress_time_low = 60.0; + double total_compress_time = 0.0; + double total_decompress_time = 0.0; + double total_time = 0.0; + double decompress_speed = 0.0; + double decompress_speed_low = 0.0; + double decompress_speed_high = 0.0; + double compress_time_elapsed = 0.0; + double decompress_time_elapsed = 0.0; + cputime_chronometer chrono; + + while (total_time <= 10.0) { + ++iterations; + + cputime_chronometer_start(&chrono); + density_compress(in, uncompressed_size, out, memory_allocated, compression_mode); + compress_time_elapsed = cputime_chronometer_stop(&chrono); + + if (!compression_only) { + cputime_chronometer_start(&chrono); + density_decompress(out, compressed_size, in, memory_allocated); + decompress_time_elapsed = cputime_chronometer_stop(&chrono); + } + + total_compress_time += compress_time_elapsed; + + if (compress_time_elapsed < compress_time_low) + compress_time_low = compress_time_elapsed; + if (compress_time_elapsed > compress_time_high) + compress_time_high = compress_time_elapsed; + + double compress_speed = ((1.0 * uncompressed_size * iterations) / (total_compress_time * 1000.0 * 1000.0)); + double compress_speed_low = ((1.0 * uncompressed_size) / (compress_time_high * 1000.0 * 1000.0)); + double compress_speed_high = ((1.0 * uncompressed_size) / (compress_time_low * 1000.0 * 1000.0)); + + total_time += compress_time_elapsed; + + if (!compression_only) { + total_decompress_time += decompress_time_elapsed; + + if (decompress_time_elapsed < decompress_time_low) + decompress_time_low = decompress_time_elapsed; + if (decompress_time_elapsed > decompress_time_high) + decompress_time_high = decompress_time_elapsed; + + decompress_speed = ((1.0 * uncompressed_size * iterations) / (total_decompress_time * 1000.0 * 1000.0)); + decompress_speed_low = ((1.0 * uncompressed_size) / (decompress_time_high * 1000.0 * 1000.0)); + decompress_speed_high = ((1.0 * uncompressed_size) / (decompress_time_low * 1000.0 * 1000.0)); + + total_time += decompress_time_elapsed; + } + + DENSITY_BENCHMARK_BLUE(printf("\rCompress speed "); + DENSITY_BENCHMARK_BOLD(printf("%.0lf MB/s", compress_speed))); + printf(" (min %.0lf MB/s, max %.0lf MB/s, best %.4lfs) ", compress_speed_low, compress_speed_high, compress_time_low); + + if (!compression_only) { + printf("<=> "); + DENSITY_BENCHMARK_BLUE(printf("Decompress speed "); + DENSITY_BENCHMARK_BOLD(printf("%.0lf MB/s", decompress_speed))); + printf(" (min %.0lf MB/s, max %.0lf MB/s, best %.4lfs) ", decompress_speed_low, decompress_speed_high, decompress_time_low); + } + fflush(stdout); + } + printf("\nRun time %.3lfs (%i iterations)\n\n", total_time, iterations); + } + + free(in); + free(out); + + printf("Allocated memory released.\n\n"); + + return EXIT_SUCCESS; +} diff --git a/contrib/density/benchmark/src/benchmark.h b/contrib/density/benchmark/src/benchmark.h new file mode 100644 index 00000000000..749e1ccb1d1 --- /dev/null +++ b/contrib/density/benchmark/src/benchmark.h @@ -0,0 +1,129 @@ +/* + * Density benchmark + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * 6/04/15 0:11 + */ + +#ifndef DENSITY_BENCHMARK_H +#define DENSITY_BENCHMARK_H + +#define _CRT_SECURE_NO_DEPRECATE + +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../src/density_api.h" +#include "../libs/cputime/src/cputime_api.h" +#include "../libs/spookyhash/src/spookyhash_api.h" + +#if defined(_WIN64) || defined(_WIN32) +#else +#define DENSITY_BENCHMARK_ALLOW_ANSI_ESCAPE_SEQUENCES +#endif + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define DENSITY_BENCHMARK_ENDIAN_STRING "Little" +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define DENSITY_BENCHMARK_ENDIAN_STRING "Big" +#endif + +#if defined(__clang__) +#define DENSITY_BENCHMARK_COMPILER "Clang %d.%d.%d" +#define DENSITY_BENCHMARK_COMPILER_VERSION __clang_major__, __clang_minor__, __clang_patchlevel__ +#elif defined(__GNUC__) +#define DENSITY_BENCHMARK_COMPILER "GCC %d.%d.%d" +#define DENSITY_BENCHMARK_COMPILER_VERSION __GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__ +#elif defined(_MSC_VER) +#define DENSITY_BENCHMARK_COMPILER "MSVC" +#define DENSITY_BENCHMARK_COMPILER_VERSION "" +#elif defined(__INTEL_COMPILER) +#define DENSITY_BENCHMARK_COMPILER "ICC" +#define DENSITY_BENCHMARK_COMPILER_VERSION "" +#else +#define DENSITY_BENCHMARK_COMPILER "an unknown compiler" +#define DENSITY_BENCHMARK_COMPILER_VERSION "" +#endif + +#if defined(_WIN64) || defined(_WIN32) +#define DENSITY_BENCHMARK_PLATFORM_STRING "Microsoft Windows" +#elif defined(__APPLE__) +#include "TargetConditionals.h" +#if TARGET_IPHONE_SIMULATOR +#define DENSITY_BENCHMARK_PLATFORM_STRING "iOS Simulator" +#elif TARGET_OS_IPHONE +#define DENSITY_BENCHMARK_PLATFORM_STRING "iOS" +#elif TARGET_OS_MAC +#define DENSITY_BENCHMARK_PLATFORM_STRING "MacOS" +#else +#define DENSITY_BENCHMARK_PLATFORM_STRING "an unknown Apple platform" +#endif +#elif defined(__FreeBSD__) +#define DENSITY_BENCHMARK_PLATFORM_STRING "FreeBSD" +#elif defined(__linux__) +#define DENSITY_BENCHMARK_PLATFORM_STRING "GNU/Linux" +#elif defined(__unix__) +#define DENSITY_BENCHMARK_PLATFORM_STRING "Unix" +#elif defined(__posix__) +#define DENSITY_BENCHMARK_PLATFORM_STRING "Posix" +#else +#define DENSITY_BENCHMARK_PLATFORM_STRING "an unknown platform" +#endif + +#define DENSITY_ESCAPE_CHARACTER ((char)27) + +#ifdef DENSITY_BENCHMARK_ALLOW_ANSI_ESCAPE_SEQUENCES +#define DENSITY_BENCHMARK_BOLD(op) printf("%c[1m", DENSITY_ESCAPE_CHARACTER);\ + op;\ + printf("%c[0m", DENSITY_ESCAPE_CHARACTER); + +#define DENSITY_BENCHMARK_BLUE(op) printf("%c[0;34m", DENSITY_ESCAPE_CHARACTER);\ + op;\ + printf("%c[0m", DENSITY_ESCAPE_CHARACTER); + +#define DENSITY_BENCHMARK_RED(op) printf("%c[0;31m", DENSITY_ESCAPE_CHARACTER);\ + op;\ + printf("%c[0m", DENSITY_ESCAPE_CHARACTER); +#else +#define DENSITY_BENCHMARK_BOLD(op) op; +#define DENSITY_BENCHMARK_BLUE(op) op; +#define DENSITY_BENCHMARK_RED(op) op; +#endif + +#define DENSITY_BENCHMARK_UNDERLINE(n) printf("\n");\ + for(int i = 0; i < n; i++) printf("="); + +#define DENSITY_BENCHMARK_ERROR(op, issue) DENSITY_BENCHMARK_RED(DENSITY_BENCHMARK_BOLD(printf("\nAn error has occured !\n")));\ + op;\ + printf("\n");\ + if(issue) {\ + printf("Please open an issue at , with your platform information and any relevant file.\n");\ + DENSITY_BENCHMARK_BOLD(printf("Thank you !\n"));\ + }\ + fflush(stdout);\ + exit(EXIT_FAILURE); + +#endif + +#define DENSITY_BENCHMARK_HASH_SEED_1 0x0123456789abcdefllu +#define DENSITY_BENCHMARK_HASH_SEED_2 0xfedcba9876543210llu diff --git a/contrib/density/msvc/Density.sln b/contrib/density/msvc/Density.sln new file mode 100644 index 00000000000..70c25ab3413 --- /dev/null +++ b/contrib/density/msvc/Density.sln @@ -0,0 +1,21 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "density", "density.vcxproj", "{65C51F09-D1A4-9EA4-DABC-297B461B0506}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "benchmark", "benchmark.vcxproj", "{7000C5C1-DC6A-7938-25A9-2ADE9152578D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {65C51F09-D1A4-9EA4-DABC-297B461B0506}.Release|x64.ActiveCfg = Release|x64 + {65C51F09-D1A4-9EA4-DABC-297B461B0506}.Release|x64.Build.0 = Release|x64 + {7000C5C1-DC6A-7938-25A9-2ADE9152578D}.Release|x64.ActiveCfg = Release|x64 + {7000C5C1-DC6A-7938-25A9-2ADE9152578D}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/contrib/density/msvc/benchmark.vcxproj b/contrib/density/msvc/benchmark.vcxproj new file mode 100644 index 00000000000..ef32bbb5f3c --- /dev/null +++ b/contrib/density/msvc/benchmark.vcxproj @@ -0,0 +1,78 @@ + + + + + Release + x64 + + + + {7000C5C1-DC6A-7938-25A9-2ADE9152578D} + true + Win32Proj + benchmark + + + + Application + false + Unicode + v140 + true + + + + + + + + + + false + bin\Release\ + obj\Release\benchmark\ + benchmark + .exe + + + + NotUsing + Level4 + MaxSpeed + true + true + false + true + true + + + Console + true + true + + + + + + + + + + + + + + + + + + + + + {65C51F09-D1A4-9EA4-DABC-297B461B0506} + + + + + + \ No newline at end of file diff --git a/contrib/density/msvc/benchmark.vcxproj.filters b/contrib/density/msvc/benchmark.vcxproj.filters new file mode 100644 index 00000000000..26379e28724 --- /dev/null +++ b/contrib/density/msvc/benchmark.vcxproj.filters @@ -0,0 +1,63 @@ + + + + + {2F149A7C-1B4B-9B0D-C437-8110B04D170F} + + + {95C9EAC4-812C-7A69-2AB2-B21F16F445EC} + + + {2C4ACB69-1843-EABB-4175-CF402DCDC9C7} + + + {C78AF53E-3316-6303-3C27-E6F7A831BF03} + + + {5E54D780-CAB5-B48E-5323-FB40BF83EB4D} + + + {2DAB880B-99B4-887C-2230-9F7C8E38947C} + + + + + libs\cputime\src + + + libs\cputime\src + + + libs\spookyhash\src + + + libs\spookyhash\src + + + libs\spookyhash\src + + + libs\spookyhash\src + + + src + + + + + libs\cputime\src + + + libs\spookyhash\src + + + libs\spookyhash\src + + + libs\spookyhash\src + + + src + + + \ No newline at end of file diff --git a/contrib/density/msvc/density.vcxproj b/contrib/density/msvc/density.vcxproj new file mode 100644 index 00000000000..f1ad77fbc1b --- /dev/null +++ b/contrib/density/msvc/density.vcxproj @@ -0,0 +1,93 @@ + + + + + Release + x64 + + + + {65C51F09-D1A4-9EA4-DABC-297B461B0506} + true + Win32Proj + density + + + + DynamicLibrary + false + Unicode + v140 + true + + + + + + + + + + false + bin\Release\ + obj\Release\density\ + density + .dll + + + + NotUsing + Level4 + MaxSpeed + true + true + false + true + true + + + Windows + true + true + bin\Release\density.lib + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/contrib/density/msvc/density.vcxproj.filters b/contrib/density/msvc/density.vcxproj.filters new file mode 100644 index 00000000000..202f2522bbe --- /dev/null +++ b/contrib/density/msvc/density.vcxproj.filters @@ -0,0 +1,135 @@ + + + + + {DF10C562-CBC8-06B0-34D0-DF0B20A7A1A0} + + + {5AB95D49-4648-E712-EF66-FB0DDBD4F7B8} + + + {1252540B-7E14-8895-C7C4-162233C66073} + + + {3F6054BC-AB43-63FC-B446-913820A9294D} + + + {C09C0CB6-AC80-CD0B-15E1-C75E01E4B78D} + + + {3845B705-A47C-FF9C-AD86-63C0193D2696} + + + {E597066D-51B0-ED96-1A5D-7D3086348230} + + + {40FA2C44-AC85-9A08-B596-1DFD21A1F608} + + + {B8122E82-A4A1-B74B-4DC0-CB46392EC8F1} + + + {6585C1C9-5195-6D9A-BA4E-D2B8A67D5C33} + + + {B6B426C8-2221-E2C2-EB14-7A205740042B} + + + {B2DFE593-1EBF-642F-27D7-EF059335CB90} + + + {367A73B3-A2E4-272A-EB22-D9CF57CC057F} + + + + + algorithms + + + algorithms\chameleon + + + algorithms\chameleon\core + + + algorithms\chameleon\core + + + algorithms\chameleon\dictionary + + + algorithms\cheetah + + + algorithms\cheetah\core + + + algorithms\cheetah\core + + + algorithms\cheetah\dictionary + + + algorithms + + + algorithms\lion\core + + + algorithms\lion\core + + + algorithms\lion\dictionary + + + algorithms\lion\forms + + + algorithms\lion + + + buffers + + + + + structure + + + + + algorithms + + + algorithms\chameleon\core + + + algorithms\chameleon\core + + + algorithms\cheetah\core + + + algorithms\cheetah\core + + + algorithms + + + algorithms\lion\core + + + algorithms\lion\core + + + algorithms\lion\forms + + + buffers + + + + structure + + + \ No newline at end of file diff --git a/contrib/density/src/algorithms/algorithms.c b/contrib/density/src/algorithms/algorithms.c new file mode 100644 index 00000000000..f36e94cd08b --- /dev/null +++ b/contrib/density/src/algorithms/algorithms.c @@ -0,0 +1,43 @@ +/* + * Density + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 14/10/15 02:06 + */ + +#include "algorithms.h" + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_algorithms_prepare_state(density_algorithm_state *const DENSITY_RESTRICT state, void *const DENSITY_RESTRICT dictionary) { + state->dictionary = dictionary; + state->copy_penalty = 0; + state->copy_penalty_start = 1; + state->previous_incompressible = false; + state->counter = 0; +} diff --git a/contrib/density/src/algorithms/algorithms.h b/contrib/density/src/algorithms/algorithms.h new file mode 100644 index 00000000000..b5e31377004 --- /dev/null +++ b/contrib/density/src/algorithms/algorithms.h @@ -0,0 +1,78 @@ +/* + * Density + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 3/02/15 19:51 + */ + +#ifndef DENSITY_ALGORITHMS_H +#define DENSITY_ALGORITHMS_H + +#include "../globals.h" + +typedef enum { + DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED = 0, + DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING, + DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL, + DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL +} density_algorithm_exit_status; + +typedef struct { + void *dictionary; + uint_fast8_t copy_penalty; + uint_fast8_t copy_penalty_start; + bool previous_incompressible; + uint_fast64_t counter; +} density_algorithm_state; + +#define DENSITY_ALGORITHM_COPY(work_block_size)\ + DENSITY_MEMCPY(*out, *in, work_block_size);\ + *in += work_block_size;\ + *out += work_block_size; + +#define DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START\ + if(!(--state->copy_penalty))\ + state->copy_penalty_start++; + +#define DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START\ + if (state->copy_penalty_start & ~0x1)\ + state->copy_penalty_start >>= 1; + +#define DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY(span, work_block_size)\ + if (DENSITY_UNLIKELY(span & ~(work_block_size - 1))) {\ + if (state->previous_incompressible)\ + state->copy_penalty = state->copy_penalty_start;\ + state->previous_incompressible = true;\ + } else\ + state->previous_incompressible = false; + +DENSITY_WINDOWS_EXPORT void density_algorithms_prepare_state(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, void *const DENSITY_RESTRICT_DECLARE); + +#endif diff --git a/contrib/density/src/algorithms/chameleon/chameleon.h b/contrib/density/src/algorithms/chameleon/chameleon.h new file mode 100644 index 00000000000..2054f34cfd8 --- /dev/null +++ b/contrib/density/src/algorithms/chameleon/chameleon.h @@ -0,0 +1,70 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 24/10/13 11:57 + * + * ------------------- + * Chameleon algorithm + * ------------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Hash based superfast kernel + */ + +#ifndef DENSITY_CHAMELEON_H +#define DENSITY_CHAMELEON_H + +#include "../../globals.h" + +#define DENSITY_CHAMELEON_HASH_BITS 16 +#define DENSITY_CHAMELEON_HASH_MULTIPLIER (uint32_t)0x9D6EF916lu + +#define DENSITY_CHAMELEON_HASH_ALGORITHM(value32) (uint16_t)((value32 * DENSITY_CHAMELEON_HASH_MULTIPLIER) >> (32 - DENSITY_CHAMELEON_HASH_BITS)) + +typedef enum { + DENSITY_CHAMELEON_SIGNATURE_FLAG_CHUNK = 0x0, + DENSITY_CHAMELEON_SIGNATURE_FLAG_MAP = 0x1, +} DENSITY_CHAMELEON_SIGNATURE_FLAG; + +typedef uint64_t density_chameleon_signature; + +#define DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE (density_bitsizeof(density_chameleon_signature) * sizeof(uint32_t)) // Uncompressed chunks +#define DENSITY_CHAMELEON_DECOMPRESSED_BODY_SIZE_PER_SIGNATURE (density_bitsizeof(density_chameleon_signature) * sizeof(uint32_t)) + +#define DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE (sizeof(density_chameleon_signature) + DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE) +#define DENSITY_CHAMELEON_DECOMPRESSED_UNIT_SIZE (DENSITY_CHAMELEON_DECOMPRESSED_BODY_SIZE_PER_SIGNATURE) + +#define DENSITY_CHAMELEON_WORK_BLOCK_SIZE 256 + +#endif diff --git a/contrib/density/src/algorithms/chameleon/core/chameleon_decode.c b/contrib/density/src/algorithms/chameleon/core/chameleon_decode.c new file mode 100644 index 00000000000..240aaba8598 --- /dev/null +++ b/contrib/density/src/algorithms/chameleon/core/chameleon_decode.c @@ -0,0 +1,254 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 23/06/15 22:11 + * + * ------------------- + * Chameleon algorithm + * ------------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Hash based superfast kernel + */ + +#include "chameleon_decode.h" + +DENSITY_FORCE_INLINE void density_chameleon_decode_process_compressed(const uint16_t hash, uint8_t **DENSITY_RESTRICT out, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) { + DENSITY_MEMCPY(*out, &dictionary->entries[hash].as_uint32_t, sizeof(uint32_t)); +} + +DENSITY_FORCE_INLINE void density_chameleon_decode_process_uncompressed(const uint32_t chunk, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) { + const uint16_t hash = DENSITY_CHAMELEON_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(chunk)); + (&dictionary->entries[hash])->as_uint32_t = chunk; // Does not ensure dictionary content consistency between endiannesses +} + +DENSITY_FORCE_INLINE void density_chameleon_decode_kernel(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const density_bool compressed, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) { + if (compressed) { + uint16_t hash; + DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t)); + density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16(hash), out, dictionary); + *in += sizeof(uint16_t); + } else { + uint32_t unit; + DENSITY_MEMCPY(&unit, *in, sizeof(uint32_t)); + density_chameleon_decode_process_uncompressed(unit, dictionary); + DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t)); + *in += sizeof(uint32_t); + } + *out += sizeof(uint32_t); +} + +DENSITY_FORCE_INLINE void density_chameleon_decode_kernel_dual(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const density_chameleon_signature signature, const uint_fast8_t shift, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) { + uint32_t var_32; + uint64_t var_64; + + switch((signature >> shift) & 0x3) { + case 0x0: + DENSITY_MEMCPY(&var_64, *in, sizeof(uint32_t) + sizeof(uint32_t)); +#ifdef DENSITY_LITTLE_ENDIAN + density_chameleon_decode_process_uncompressed((uint32_t)(var_64 & 0xffffffff), dictionary); +#endif + density_chameleon_decode_process_uncompressed((uint32_t)(var_64 >> density_bitsizeof(uint32_t)), dictionary); +#ifdef DENSITY_BIG_ENDIAN + density_chameleon_decode_process_uncompressed((uint32_t)(var_64 & 0xffffffff), dictionary); +#endif + DENSITY_MEMCPY(*out, &var_64, sizeof(uint32_t) + sizeof(uint32_t)); + *in += (sizeof(uint32_t) + sizeof(uint32_t)); + *out += sizeof(uint64_t); + break; + case 0x1: + DENSITY_MEMCPY(&var_64, *in, sizeof(uint16_t) + sizeof(uint32_t)); +#ifdef DENSITY_LITTLE_ENDIAN + density_chameleon_decode_process_compressed((uint16_t)(var_64 & 0xffff), out, dictionary); + var_32 = (uint32_t)((var_64 >> density_bitsizeof(uint16_t)) & 0xffffffff); + density_chameleon_decode_process_uncompressed(var_32, dictionary); + DENSITY_MEMCPY(*out + sizeof(uint32_t), &var_32, sizeof(uint32_t)); + *out += sizeof(uint64_t); +#elif defined(DENSITY_BIG_ENDIAN) + density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16((uint16_t)((var_64 >> (density_bitsizeof(uint16_t) + density_bitsizeof(uint32_t))) & 0xffff)), out, dictionary); + var_32 = (uint32_t)((var_64 >> density_bitsizeof(uint16_t)) & 0xffffffff); + density_chameleon_decode_process_uncompressed(var_32, dictionary); + DENSITY_MEMCPY(*out + sizeof(uint32_t), &var_32, sizeof(uint32_t)); + *out += sizeof(uint64_t); +#else +#error +#endif + *in += (sizeof(uint16_t) + sizeof(uint32_t)); + break; + case 0x2: + DENSITY_MEMCPY(&var_64, *in, sizeof(uint32_t) + sizeof(uint16_t)); +#ifdef DENSITY_LITTLE_ENDIAN + var_32 = (uint32_t)(var_64 & 0xffffffff); + density_chameleon_decode_process_uncompressed(var_32, dictionary); + DENSITY_MEMCPY(*out, &var_32, sizeof(uint32_t)); + *out += sizeof(uint32_t); + density_chameleon_decode_process_compressed((uint16_t)((var_64 >> density_bitsizeof(uint32_t)) & 0xffff), out, dictionary); + *out += sizeof(uint32_t); +#elif defined(DENSITY_BIG_ENDIAN) + var_32 = (uint32_t)((var_64 >> density_bitsizeof(uint32_t)) & 0xffffffff); + density_chameleon_decode_process_uncompressed(var_32, dictionary); + DENSITY_MEMCPY(*out, &var_32, sizeof(uint32_t)); + *out += sizeof(uint32_t); + density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16((uint16_t)((var_64 >> density_bitsizeof(uint16_t)) & 0xffff)), out, dictionary); + *out += sizeof(uint32_t); +#else +#error +#endif + *in += (sizeof(uint32_t) + sizeof(uint16_t)); + break; + case 0x3: + DENSITY_MEMCPY(&var_32, *in, sizeof(uint16_t) + sizeof(uint16_t)); +#ifdef DENSITY_LITTLE_ENDIAN + density_chameleon_decode_process_compressed((uint16_t)(var_32 & 0xffff), out, dictionary); + *out += sizeof(uint32_t); +#endif + density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16((uint16_t)(var_32 >> density_bitsizeof(uint16_t))), out, dictionary); + *out += sizeof(uint32_t); +#ifdef DENSITY_BIG_ENDIAN + density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16((uint16_t)(var_32 & 0xffff)), out, dictionary); + *out += sizeof(uint32_t); +#endif + *in += (sizeof(uint16_t) + sizeof(uint16_t)); + break; + } +} + +DENSITY_FORCE_INLINE bool density_chameleon_decode_test_compressed(const density_chameleon_signature signature, const uint_fast8_t shift) { + return (density_bool const) ((signature >> shift) & DENSITY_CHAMELEON_SIGNATURE_FLAG_MAP); +} + +DENSITY_FORCE_INLINE void density_chameleon_decode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const density_chameleon_signature signature, const uint_fast8_t shift, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) { + density_chameleon_decode_kernel(in, out, density_chameleon_decode_test_compressed(signature, shift), dictionary); +} + +DENSITY_FORCE_INLINE void density_chameleon_decode_256(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const density_chameleon_signature signature, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) { + uint_fast8_t count_a = 0; + uint_fast8_t count_b = 0; + +#if defined(__clang__) || defined(_MSC_VER) + do { + DENSITY_UNROLL_2(density_chameleon_decode_kernel_dual(in, out, signature, count_a, dictionary); count_a+= 2); + } while (++count_b & 0xf); +#else + do { + DENSITY_UNROLL_2(density_chameleon_decode_4(in, out, signature, count_a ++, dictionary)); + } while (++count_b & 0x1f); +#endif +} + +DENSITY_FORCE_INLINE void density_chameleon_decode_read_signature(const uint8_t **DENSITY_RESTRICT in, density_chameleon_signature *DENSITY_RESTRICT signature) { +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(signature, *in, sizeof(density_chameleon_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + density_chameleon_signature endian_signature; + DENSITY_MEMCPY(&endian_signature, *in, sizeof(density_chameleon_signature)); + *signature = DENSITY_LITTLE_ENDIAN_64(endian_signature); +#else +#error +#endif + *in += sizeof(density_chameleon_signature); +} + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_chameleon_decode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) { + if (out_size < DENSITY_CHAMELEON_DECOMPRESSED_UNIT_SIZE) + return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; + + density_chameleon_signature signature; + uint_fast8_t shift; + uint_fast64_t remaining; + + const uint8_t *start = *in; + + if (in_size < DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE) { + goto read_signature; + } + + const uint8_t *in_limit = *in + in_size - DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE; + uint8_t *out_limit = *out + out_size - DENSITY_CHAMELEON_DECOMPRESSED_UNIT_SIZE; + + while (DENSITY_LIKELY(*in <= in_limit && *out <= out_limit)) { + if (DENSITY_UNLIKELY(!(state->counter & 0xf))) { + DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START; + } + state->counter++; + if (DENSITY_UNLIKELY(state->copy_penalty)) { + DENSITY_ALGORITHM_COPY(DENSITY_CHAMELEON_WORK_BLOCK_SIZE); + DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START; + } else { + const uint8_t *in_start = *in; + density_chameleon_decode_read_signature(in, &signature); + density_chameleon_decode_256(in, out, signature, (density_chameleon_dictionary *const) state->dictionary); + DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*in - in_start), DENSITY_CHAMELEON_WORK_BLOCK_SIZE); + } + } + + if (*out > out_limit) + return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; + + read_signature: + if (in_size - (*in - start) < sizeof(density_chameleon_signature)) + return DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL; + shift = 0; + density_chameleon_decode_read_signature(in, &signature); + read_and_decode_4: + switch (in_size - (*in - start)) { + case 0: + case 1: + if (density_chameleon_decode_test_compressed(signature, shift)) + return DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING; + else // End marker + goto process_remaining_bytes; + case 2: + case 3: + if (density_chameleon_decode_test_compressed(signature, shift++)) + density_chameleon_decode_kernel(in, out, true, (density_chameleon_dictionary *const) state->dictionary); + else // End marker + goto process_remaining_bytes; + break; + default: + density_chameleon_decode_4(in, out, signature, shift++, (density_chameleon_dictionary *const) state->dictionary); + break; + } + + if (DENSITY_UNLIKELY(shift == density_bitsizeof(density_chameleon_signature))) + goto read_signature; + else + goto read_and_decode_4; + + process_remaining_bytes: + remaining = in_size - (*in - start); + DENSITY_ALGORITHM_COPY(remaining); + + return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED; +} diff --git a/contrib/density/src/algorithms/chameleon/core/chameleon_decode.h b/contrib/density/src/algorithms/chameleon/core/chameleon_decode.h new file mode 100644 index 00000000000..cbf06e26a44 --- /dev/null +++ b/contrib/density/src/algorithms/chameleon/core/chameleon_decode.h @@ -0,0 +1,53 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 23/06/15 21:49 + * + * ------------------- + * Chameleon algorithm + * ------------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Hash based superfast kernel + */ + +#ifndef DENSITY_CHAMELEON_DECODE_H +#define DENSITY_CHAMELEON_DECODE_H + +#include "../dictionary/chameleon_dictionary.h" +#include "../../algorithms.h" + +DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_chameleon_decode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t); + +#endif diff --git a/contrib/density/src/algorithms/chameleon/core/chameleon_encode.c b/contrib/density/src/algorithms/chameleon/core/chameleon_encode.c new file mode 100644 index 00000000000..2090cb99173 --- /dev/null +++ b/contrib/density/src/algorithms/chameleon/core/chameleon_encode.c @@ -0,0 +1,179 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 23/06/15 22:02 + * + * ------------------- + * Chameleon algorithm + * ------------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Hash based superfast kernel + */ + +#include "chameleon_encode.h" + +DENSITY_FORCE_INLINE void density_chameleon_encode_prepare_signature(uint8_t **DENSITY_RESTRICT out, density_chameleon_signature **DENSITY_RESTRICT signature_pointer, density_chameleon_signature *const DENSITY_RESTRICT signature) { + *signature = 0; + *signature_pointer = (density_chameleon_signature *) *out; + *out += sizeof(density_chameleon_signature); +} + +DENSITY_FORCE_INLINE void density_chameleon_encode_kernel(uint8_t **DENSITY_RESTRICT out, const uint16_t hash, const uint_fast8_t shift, density_chameleon_signature *const DENSITY_RESTRICT signature, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) { + density_chameleon_dictionary_entry *const found = &dictionary->entries[hash]; + + switch (*unit ^ found->as_uint32_t) { + case 0: + *signature |= ((uint64_t) DENSITY_CHAMELEON_SIGNATURE_FLAG_MAP << shift); +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); +#elif defined(DENSITY_BIG_ENDIAN) + const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); + DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); +#else +#error +#endif + *out += sizeof(uint16_t); + break; + default: + found->as_uint32_t = *unit; // Does not ensure dictionary content consistency between endiannesses + DENSITY_MEMCPY(*out, unit, sizeof(uint32_t)); + *out += sizeof(uint32_t); + break; + } +} + +DENSITY_FORCE_INLINE void density_chameleon_encode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const uint_fast8_t shift, density_chameleon_signature *const DENSITY_RESTRICT signature, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) { + DENSITY_MEMCPY(unit, *in, sizeof(uint32_t)); + density_chameleon_encode_kernel(out, DENSITY_CHAMELEON_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)), shift, signature, dictionary, unit); + *in += sizeof(uint32_t); +} + +DENSITY_FORCE_INLINE void density_chameleon_encode_256(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, density_chameleon_signature *const DENSITY_RESTRICT signature, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) { + uint_fast8_t count = 0; + +#ifdef __clang__ + for (uint_fast8_t count_b = 0; count_b < 32; count_b++) { + DENSITY_UNROLL_2(density_chameleon_encode_4(in, out, count++, signature, dictionary, unit)); + } +#else + for (uint_fast8_t count_b = 0; count_b < 16; count_b++) { + DENSITY_UNROLL_4(density_chameleon_encode_4(in, out, count++, signature, dictionary, unit)); + } +#endif +} + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_chameleon_encode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) { + if (out_size < DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE) + return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; + + density_chameleon_signature signature; + density_chameleon_signature *signature_pointer; + uint32_t unit; + + uint8_t *out_limit = *out + out_size - DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE; + uint_fast64_t limit_256 = (in_size >> 8); + + while (DENSITY_LIKELY(limit_256-- && *out <= out_limit)) { + if (DENSITY_UNLIKELY(!(state->counter & 0xf))) { + DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START; + } + state->counter++; + if (DENSITY_UNLIKELY(state->copy_penalty)) { + DENSITY_ALGORITHM_COPY(DENSITY_CHAMELEON_WORK_BLOCK_SIZE); + DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START; + } else { + const uint8_t *out_start = *out; + density_chameleon_encode_prepare_signature(out, &signature_pointer, &signature); + DENSITY_PREFETCH(*in + DENSITY_CHAMELEON_WORK_BLOCK_SIZE); + density_chameleon_encode_256(in, out, &signature, (density_chameleon_dictionary *const) state->dictionary, &unit); +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_chameleon_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + const density_chameleon_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); + DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_chameleon_signature)); +#else +#error +#endif + DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*out - out_start), DENSITY_CHAMELEON_WORK_BLOCK_SIZE); + } + } + + if (*out > out_limit) + return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; + + uint_fast64_t remaining; + + switch (in_size & 0xff) { + case 0: + case 1: + case 2: + case 3: + density_chameleon_encode_prepare_signature(out, &signature_pointer, &signature); + signature = ((uint64_t) DENSITY_CHAMELEON_SIGNATURE_FLAG_CHUNK); // End marker +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_chameleon_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + const density_chameleon_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); + DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_chameleon_signature)); +#else +#error +#endif + goto process_remaining_bytes; + default: + break; + } + + const uint_fast64_t limit_4 = (in_size & 0xff) >> 2; + density_chameleon_encode_prepare_signature(out, &signature_pointer, &signature); + for (uint_fast8_t shift = 0; shift != limit_4; shift++) + density_chameleon_encode_4(in, out, shift, &signature, (density_chameleon_dictionary *const) state->dictionary, &unit); + + signature |= ((uint64_t) DENSITY_CHAMELEON_SIGNATURE_FLAG_CHUNK << limit_4); // End marker +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_chameleon_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + const density_chameleon_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); + DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_chameleon_signature)); +#else +#error +#endif + + process_remaining_bytes: + remaining = in_size & 0x3; + if (remaining) + DENSITY_ALGORITHM_COPY(remaining); + + return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED; +} diff --git a/contrib/density/src/algorithms/chameleon/core/chameleon_encode.h b/contrib/density/src/algorithms/chameleon/core/chameleon_encode.h new file mode 100644 index 00000000000..f71fc23fbc0 --- /dev/null +++ b/contrib/density/src/algorithms/chameleon/core/chameleon_encode.h @@ -0,0 +1,53 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 23/06/15 21:51 + * + * ------------------- + * Chameleon algorithm + * ------------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Hash based superfast kernel + */ + +#ifndef DENSITY_CHAMELEON_ENCODE_H +#define DENSITY_CHAMELEON_ENCODE_H + +#include "../dictionary/chameleon_dictionary.h" +#include "../../algorithms.h" + +DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_chameleon_encode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t); + +#endif diff --git a/contrib/density/src/algorithms/chameleon/dictionary/chameleon_dictionary.h b/contrib/density/src/algorithms/chameleon/dictionary/chameleon_dictionary.h new file mode 100644 index 00000000000..1f584a5936e --- /dev/null +++ b/contrib/density/src/algorithms/chameleon/dictionary/chameleon_dictionary.h @@ -0,0 +1,63 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 24/10/13 12:05 + * + * ------------------- + * Chameleon algorithm + * ------------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Hash based superfast kernel + */ + +#ifndef DENSITY_CHAMELEON_DICTIONARY_H +#define DENSITY_CHAMELEON_DICTIONARY_H + +#include "../chameleon.h" + +#include + +#pragma pack(push) +#pragma pack(4) +typedef struct { + uint32_t as_uint32_t; +} density_chameleon_dictionary_entry; + +typedef struct { + density_chameleon_dictionary_entry entries[1 << DENSITY_CHAMELEON_HASH_BITS]; +} density_chameleon_dictionary; +#pragma pack(pop) + +#endif diff --git a/contrib/density/src/algorithms/cheetah/cheetah.h b/contrib/density/src/algorithms/cheetah/cheetah.h new file mode 100644 index 00000000000..0d47d0ab502 --- /dev/null +++ b/contrib/density/src/algorithms/cheetah/cheetah.h @@ -0,0 +1,73 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 06/12/13 20:10 + * + * ----------------- + * Cheetah algorithm + * ----------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * Piotr Tarsa (https://github.com/tarsa) + * + * Description + * Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup + */ + +#ifndef DENSITY_CHEETAH_H +#define DENSITY_CHEETAH_H + +#include "../../globals.h" + +#define DENSITY_CHEETAH_HASH_BITS 16 +#define DENSITY_CHEETAH_HASH_MULTIPLIER (uint32_t)0x9D6EF916lu + +#define DENSITY_CHEETAH_HASH_ALGORITHM(value32) (uint16_t)(value32 * DENSITY_CHEETAH_HASH_MULTIPLIER >> (32 - DENSITY_CHEETAH_HASH_BITS)) + +typedef enum { + DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED = 0x0, + DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_A = 0x1, + DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_B = 0x2, + DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK = 0x3, +} DENSITY_CHEETAH_SIGNATURE_FLAG; + +typedef uint64_t density_cheetah_signature; + +#define DENSITY_CHEETAH_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE ((density_bitsizeof(density_cheetah_signature) >> 1) * sizeof(uint32_t)) // Uncompressed chunks +#define DENSITY_CHEETAH_DECOMPRESSED_BODY_SIZE_PER_SIGNATURE ((density_bitsizeof(density_cheetah_signature) >> 1) * sizeof(uint32_t)) + +#define DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE (sizeof(density_cheetah_signature) + DENSITY_CHEETAH_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE) +#define DENSITY_CHEETAH_DECOMPRESSED_UNIT_SIZE (DENSITY_CHEETAH_DECOMPRESSED_BODY_SIZE_PER_SIGNATURE) + +#define DENSITY_CHEETAH_WORK_BLOCK_SIZE 128 + +#endif \ No newline at end of file diff --git a/contrib/density/src/algorithms/cheetah/core/cheetah_decode.c b/contrib/density/src/algorithms/cheetah/core/cheetah_decode.c new file mode 100644 index 00000000000..4928938b41e --- /dev/null +++ b/contrib/density/src/algorithms/cheetah/core/cheetah_decode.c @@ -0,0 +1,266 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 24/06/15 0:32 + * + * ----------------- + * Cheetah algorithm + * ----------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * Piotr Tarsa (https://github.com/tarsa) + * + * Description + * Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup + */ + +#include "cheetah_decode.h" + +DENSITY_FORCE_INLINE void density_cheetah_decode_process_predicted(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) { + const uint32_t unit = dictionary->prediction_entries[*last_hash].next_chunk_prediction; + DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t)); + *last_hash = DENSITY_CHEETAH_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(unit)); +} + +DENSITY_FORCE_INLINE void density_cheetah_decode_process_compressed_a(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, const uint16_t hash) { + DENSITY_PREFETCH(&dictionary->prediction_entries[hash]); + const uint32_t unit = dictionary->entries[hash].chunk_a; + DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t)); + dictionary->prediction_entries[*last_hash].next_chunk_prediction = unit; + *last_hash = hash; +} + +DENSITY_FORCE_INLINE void density_cheetah_decode_process_compressed_b(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, const uint16_t hash) { + DENSITY_PREFETCH(&dictionary->prediction_entries[hash]); + density_cheetah_dictionary_entry *const entry = &dictionary->entries[hash]; + const uint32_t unit = entry->chunk_b; + entry->chunk_b = entry->chunk_a; + entry->chunk_a = unit; // Does not ensure dictionary content consistency between endiannesses + DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t)); + dictionary->prediction_entries[*last_hash].next_chunk_prediction = unit; + *last_hash = hash; +} + +DENSITY_FORCE_INLINE void density_cheetah_decode_process_uncompressed(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, const uint32_t unit) { + const uint16_t hash = DENSITY_CHEETAH_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(unit)); + DENSITY_PREFETCH(&dictionary->prediction_entries[hash]); + density_cheetah_dictionary_entry *const entry = &dictionary->entries[hash]; + entry->chunk_b = entry->chunk_a; + entry->chunk_a = unit; // Does not ensure dictionary content consistency between endiannesses + DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t)); + dictionary->prediction_entries[*last_hash].next_chunk_prediction = unit; // Does not ensure dictionary content consistency between endiannesses + *last_hash = hash; +} + +DENSITY_FORCE_INLINE void density_cheetah_decode_kernel_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const uint8_t flag, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) { + uint16_t hash; + uint32_t unit; + + switch (flag) { + case DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED: + density_cheetah_decode_process_predicted(out, last_hash, dictionary); + break; + case DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_A: + DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t)); + density_cheetah_decode_process_compressed_a(out, last_hash, dictionary, DENSITY_LITTLE_ENDIAN_16(hash)); + *in += sizeof(uint16_t); + break; + case DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_B: + DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t)); + density_cheetah_decode_process_compressed_b(out, last_hash, dictionary, DENSITY_LITTLE_ENDIAN_16(hash)); + *in += sizeof(uint16_t); + break; + default: // DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK + DENSITY_MEMCPY(&unit, *in, sizeof(uint32_t)); + density_cheetah_decode_process_uncompressed(out, last_hash, dictionary, unit); + *in += sizeof(uint32_t); + break; + } + + *out += sizeof(uint32_t); +} + +DENSITY_FORCE_INLINE void density_cheetah_decode_kernel_16(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const uint8_t flags, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) { + uint16_t hash; + uint32_t unit; + + switch (flags) { + DENSITY_CASE_GENERATOR_4_4_COMBINED(\ + density_cheetah_decode_process_predicted(out, last_hash, dictionary);, \ + DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED, \ + DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t)); \ + density_cheetah_decode_process_compressed_a(out, last_hash, dictionary, DENSITY_LITTLE_ENDIAN_16(hash));\ + *in += sizeof(uint16_t);, \ + DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_A, \ + DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t)); \ + density_cheetah_decode_process_compressed_b(out, last_hash, dictionary, DENSITY_LITTLE_ENDIAN_16(hash));\ + *in += sizeof(uint16_t);, \ + DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_B, \ + DENSITY_MEMCPY(&unit, *in, sizeof(uint32_t)); \ + density_cheetah_decode_process_uncompressed(out, last_hash, dictionary, unit);\ + *in += sizeof(uint32_t);, \ + DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK, \ + *out += sizeof(uint32_t);, \ + 2\ + ); + default: + break; + } + + *out += sizeof(uint32_t); +} + +DENSITY_FORCE_INLINE uint8_t density_cheetah_decode_read_flag(const density_cheetah_signature signature, const uint_fast8_t shift) { + return (uint8_t const) ((signature >> shift) & 0x3); +} + +DENSITY_FORCE_INLINE void density_cheetah_decode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const density_cheetah_signature signature, const uint_fast8_t shift, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) { + density_cheetah_decode_kernel_4(in, out, last_hash, density_cheetah_decode_read_flag(signature, shift), dictionary); +} + +DENSITY_FORCE_INLINE void density_cheetah_decode_16(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const density_cheetah_signature signature, const uint_fast8_t shift, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) { + density_cheetah_decode_kernel_16(in, out, last_hash, (uint8_t const) ((signature >> shift) & 0xff), dictionary); +} + +DENSITY_FORCE_INLINE void density_cheetah_decode_128(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const density_cheetah_signature signature, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) { +#ifdef __clang__ + uint_fast8_t count = 0; + for (uint_fast8_t count_b = 0; count_b < 8; count_b ++) { + density_cheetah_decode_16(in, out, last_hash, signature, count, dictionary); + count += 8; + } +#else + for (uint_fast8_t count_b = 0; count_b < density_bitsizeof(density_cheetah_signature); count_b += 8) + density_cheetah_decode_16(in, out, last_hash, signature, count_b, dictionary); +#endif +} + +DENSITY_FORCE_INLINE void density_cheetah_decode_read_signature(const uint8_t **DENSITY_RESTRICT in, density_cheetah_signature *DENSITY_RESTRICT signature) { +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(signature, *in, sizeof(density_cheetah_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + density_cheetah_signature endian_signature; + DENSITY_MEMCPY(&endian_signature, *in, sizeof(density_cheetah_signature)); + *signature = DENSITY_LITTLE_ENDIAN_64(endian_signature); +#else +#error +#endif + *in += sizeof(density_cheetah_signature); +} + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_cheetah_decode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) { + if (out_size < DENSITY_CHEETAH_DECOMPRESSED_UNIT_SIZE) + return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; + + density_cheetah_signature signature; + uint_fast8_t shift; + uint_fast64_t remaining; + uint_fast16_t last_hash = 0; + uint8_t flag; + + const uint8_t *start = *in; + + if (in_size < DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE) { + goto read_signature; + } + + const uint8_t *in_limit = *in + in_size - DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE; + uint8_t *out_limit = *out + out_size - DENSITY_CHEETAH_DECOMPRESSED_UNIT_SIZE; + + while (DENSITY_LIKELY(*in <= in_limit && *out <= out_limit)) { + if (DENSITY_UNLIKELY(!(state->counter & 0x1f))) { + DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START; + } + state->counter++; + if (DENSITY_UNLIKELY(state->copy_penalty)) { + DENSITY_ALGORITHM_COPY(DENSITY_CHEETAH_WORK_BLOCK_SIZE); + DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START; + } else { + const uint8_t *in_start = *in; + density_cheetah_decode_read_signature(in, &signature); + density_cheetah_decode_128(in, out, &last_hash, signature, (density_cheetah_dictionary *const) state->dictionary); + DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*in - in_start), DENSITY_CHEETAH_WORK_BLOCK_SIZE); + } + } + + if (*out > out_limit) + return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; + + read_signature: + if (in_size - (*in - start) < sizeof(density_cheetah_signature)) + return DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL; + shift = 0; + density_cheetah_decode_read_signature(in, &signature); + read_and_decode_4: + switch (in_size - (*in - start)) { + case 0: + case 1: + switch (density_cheetah_decode_read_flag(signature, shift)) { + case DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK: + goto process_remaining_bytes; // End marker + case DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED: + density_cheetah_decode_kernel_4(in, out, &last_hash, DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED, (density_cheetah_dictionary *const) state->dictionary); + shift += 2; + break; + default: + return DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING; + } + break; + case 2: + case 3: + flag = density_cheetah_decode_read_flag(signature, shift); + switch (flag) { + case DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK: + goto process_remaining_bytes; // End marker + default: + density_cheetah_decode_kernel_4(in, out, &last_hash, flag, (density_cheetah_dictionary *const) state->dictionary); + shift += 2; + break; + } + break; + default: + density_cheetah_decode_4(in, out, &last_hash, signature, shift, (density_cheetah_dictionary *const) state->dictionary); + shift += 2; + break; + } + + if (DENSITY_UNLIKELY(shift == density_bitsizeof(density_cheetah_signature))) + goto read_signature; + else + goto read_and_decode_4; + + process_remaining_bytes: + remaining = in_size - (*in - start); + DENSITY_ALGORITHM_COPY(remaining); + + return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED; +} diff --git a/contrib/density/src/algorithms/cheetah/core/cheetah_decode.h b/contrib/density/src/algorithms/cheetah/core/cheetah_decode.h new file mode 100644 index 00000000000..278368e7938 --- /dev/null +++ b/contrib/density/src/algorithms/cheetah/core/cheetah_decode.h @@ -0,0 +1,54 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 24/06/15 0:31 + * + * ----------------- + * Cheetah algorithm + * ----------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * Piotr Tarsa (https://github.com/tarsa) + * + * Description + * Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup + */ + +#ifndef DENSITY_CHEETAH_DECODE_H +#define DENSITY_CHEETAH_DECODE_H + +#include "../dictionary/cheetah_dictionary.h" +#include "../../algorithms.h" + +DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_cheetah_decode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t); + +#endif diff --git a/contrib/density/src/algorithms/cheetah/core/cheetah_encode.c b/contrib/density/src/algorithms/cheetah/core/cheetah_encode.c new file mode 100644 index 00000000000..6cec16f4bf0 --- /dev/null +++ b/contrib/density/src/algorithms/cheetah/core/cheetah_encode.c @@ -0,0 +1,202 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 23/06/15 23:29 + * + * ----------------- + * Cheetah algorithm + * ----------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * Piotr Tarsa (https://github.com/tarsa) + * + * Description + * Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup + */ + +#include "cheetah_encode.h" + +DENSITY_FORCE_INLINE void density_cheetah_encode_prepare_signature(uint8_t **DENSITY_RESTRICT out, density_cheetah_signature **DENSITY_RESTRICT signature_pointer, density_cheetah_signature *const DENSITY_RESTRICT signature) { + *signature = 0; + *signature_pointer = (density_cheetah_signature *) *out; + *out += sizeof(density_cheetah_signature); +} + +DENSITY_FORCE_INLINE void density_cheetah_encode_kernel(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const uint_fast16_t hash, const uint_fast8_t shift, density_cheetah_signature *const DENSITY_RESTRICT signature, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) { + uint32_t *predictedChunk = (uint32_t*) &dictionary->prediction_entries[*last_hash]; + + if (*predictedChunk ^ *unit) { + density_cheetah_dictionary_entry *found = &dictionary->entries[hash]; + uint32_t *found_a = &found->chunk_a; + if (*found_a ^ *unit) { + uint32_t *found_b = &found->chunk_b; + if (*found_b ^ *unit) { + *signature |= ((uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK << shift); + DENSITY_MEMCPY(*out, unit, sizeof(uint32_t)); + *out += sizeof(uint32_t); + } else { + *signature |= ((uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_B << shift); +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); +#elif defined(DENSITY_BIG_ENDIAN) + const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); + DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); +#else +#error +#endif + *out += sizeof(uint16_t); + } + *found_b = *found_a; + *found_a = *unit; // Does not ensure dictionary content consistency between endiannesses + } else { + *signature |= ((uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_A << shift); +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); +#elif defined(DENSITY_BIG_ENDIAN) + const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); + DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); +#else +#error +#endif + *out += sizeof(uint16_t); + } + *predictedChunk = *unit; // Does not ensure dictionary content consistency between endiannesses + } + *last_hash = hash; +} + +DENSITY_FORCE_INLINE void density_cheetah_encode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const uint_fast8_t shift, density_cheetah_signature *const DENSITY_RESTRICT signature, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) { + DENSITY_MEMCPY(unit, *in, sizeof(uint32_t)); + *in += sizeof(uint32_t); + density_cheetah_encode_kernel(out, last_hash, DENSITY_CHEETAH_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)), shift, signature, dictionary, unit); +} + +DENSITY_FORCE_INLINE void density_cheetah_encode_128(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_signature *const DENSITY_RESTRICT signature, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) { + uint_fast8_t count = 0; + +#ifdef __clang__ + for(; count < density_bitsizeof(density_cheetah_signature); count += 2) { + density_cheetah_encode_4(in, out, last_hash, count, signature, dictionary, unit); + } +#else + for (uint_fast8_t count_b = 0; count_b < 16; count_b++) { + DENSITY_UNROLL_2(\ + density_cheetah_encode_4(in, out, last_hash, count, signature, dictionary, unit);\ + count += 2); + } +#endif +} + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_cheetah_encode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) { + if (out_size < DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE) + return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; + + density_cheetah_signature signature; + density_cheetah_signature *signature_pointer; + uint_fast16_t last_hash = 0; + uint32_t unit; + + uint8_t *out_limit = *out + out_size - DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE; + uint_fast64_t limit_128 = (in_size >> 7); + + while (DENSITY_LIKELY(limit_128-- && *out <= out_limit)) { + if (DENSITY_UNLIKELY(!(state->counter & 0x1f))) { + DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START; + } + state->counter++; + if (DENSITY_UNLIKELY(state->copy_penalty)) { + DENSITY_ALGORITHM_COPY(DENSITY_CHEETAH_WORK_BLOCK_SIZE); + DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START; + } else { + const uint8_t *out_start = *out; + density_cheetah_encode_prepare_signature(out, &signature_pointer, &signature); + DENSITY_PREFETCH(*in + DENSITY_CHEETAH_WORK_BLOCK_SIZE); + density_cheetah_encode_128(in, out, &last_hash, &signature, (density_cheetah_dictionary *const) state->dictionary, &unit); +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_cheetah_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + const density_cheetah_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); + DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_cheetah_signature)); +#else +#error +#endif + DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*out - out_start), DENSITY_CHEETAH_WORK_BLOCK_SIZE); + } + } + + if (*out > out_limit) + return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; + + uint_fast64_t remaining; + + switch (in_size & 0x7f) { + case 0: + case 1: + case 2: + case 3: + density_cheetah_encode_prepare_signature(out, &signature_pointer, &signature); + signature = (uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK; // End marker +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_cheetah_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + const density_cheetah_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); + DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_cheetah_signature)); +#else +#error +#endif + goto process_remaining_bytes; + default: + break; + } + + const uint_fast64_t limit_4 = ((in_size & 0x7f) >> 2) << 1; // 4-byte units times number of signature flag bits + density_cheetah_encode_prepare_signature(out, &signature_pointer, &signature); + for (uint_fast8_t shift = 0; shift != limit_4; shift += 2) + density_cheetah_encode_4(in, out, &last_hash, shift, &signature, (density_cheetah_dictionary *const) state->dictionary, &unit); + + signature |= ((uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK << limit_4); // End marker +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_cheetah_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + const density_cheetah_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); + DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_cheetah_signature)); +#else +#error +#endif + + process_remaining_bytes: + remaining = in_size & 0x3; + if (remaining) + DENSITY_ALGORITHM_COPY(remaining); + + return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED; +} diff --git a/contrib/density/src/algorithms/cheetah/core/cheetah_encode.h b/contrib/density/src/algorithms/cheetah/core/cheetah_encode.h new file mode 100644 index 00000000000..9f67e5bad9d --- /dev/null +++ b/contrib/density/src/algorithms/cheetah/core/cheetah_encode.h @@ -0,0 +1,54 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 23/06/15 23:29 + * + * ----------------- + * Cheetah algorithm + * ----------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * Piotr Tarsa (https://github.com/tarsa) + * + * Description + * Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup + */ + +#ifndef DENSITY_CHEETAH_ENCODE_H +#define DENSITY_CHEETAH_ENCODE_H + +#include "../dictionary/cheetah_dictionary.h" +#include "../../algorithms.h" + +DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_cheetah_encode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t); + +#endif diff --git a/contrib/density/src/algorithms/cheetah/dictionary/cheetah_dictionary.h b/contrib/density/src/algorithms/cheetah/dictionary/cheetah_dictionary.h new file mode 100644 index 00000000000..5223d843574 --- /dev/null +++ b/contrib/density/src/algorithms/cheetah/dictionary/cheetah_dictionary.h @@ -0,0 +1,70 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 06/12/13 20:20 + * + * ----------------- + * Cheetah algorithm + * ----------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * Piotr Tarsa (https://github.com/tarsa) + * + * Description + * Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup + */ + +#ifndef DENSITY_CHEETAH_DICTIONARY_H +#define DENSITY_CHEETAH_DICTIONARY_H + +#include "../cheetah.h" + +#include + +#pragma pack(push) +#pragma pack(4) +typedef struct { + uint32_t chunk_a; + uint32_t chunk_b; +} density_cheetah_dictionary_entry; + +typedef struct { + uint32_t next_chunk_prediction; +} density_cheetah_dictionary_prediction_entry; + +typedef struct { + density_cheetah_dictionary_entry entries[1 << DENSITY_CHEETAH_HASH_BITS]; + density_cheetah_dictionary_prediction_entry prediction_entries[1 << DENSITY_CHEETAH_HASH_BITS]; +} density_cheetah_dictionary; +#pragma pack(pop) + +#endif diff --git a/contrib/density/src/algorithms/dictionaries.c b/contrib/density/src/algorithms/dictionaries.c new file mode 100644 index 00000000000..1e8d5466014 --- /dev/null +++ b/contrib/density/src/algorithms/dictionaries.c @@ -0,0 +1,48 @@ +/* + * Density + * + * Copyright (c) 2018, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 13/01/18 17:31 + */ + +#include "dictionaries.h" + +DENSITY_WINDOWS_EXPORT size_t density_get_dictionary_size(DENSITY_ALGORITHM algorithm) { + switch(algorithm) { + case DENSITY_ALGORITHM_CHAMELEON: + return sizeof(density_chameleon_dictionary); + case DENSITY_ALGORITHM_CHEETAH: + return sizeof(density_cheetah_dictionary); + case DENSITY_ALGORITHM_LION: + return sizeof(density_lion_dictionary); + default: + return 0; + } +} diff --git a/contrib/density/src/algorithms/dictionaries.h b/contrib/density/src/algorithms/dictionaries.h new file mode 100644 index 00000000000..291c361963e --- /dev/null +++ b/contrib/density/src/algorithms/dictionaries.h @@ -0,0 +1,45 @@ +/* + * Density + * + * Copyright (c) 2018, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 13/01/18 17:26 + */ + +#ifndef DENSITY_DICTIONARIES_H +#define DENSITY_DICTIONARIES_H + +#include "../globals.h" +#include "../algorithms/chameleon/dictionary/chameleon_dictionary.h" +#include "../algorithms/cheetah/dictionary/cheetah_dictionary.h" +#include "../algorithms/lion/dictionary/lion_dictionary.h" + +DENSITY_WINDOWS_EXPORT size_t density_get_dictionary_size(DENSITY_ALGORITHM); + +#endif diff --git a/contrib/density/src/algorithms/lion/core/lion_decode.c b/contrib/density/src/algorithms/lion/core/lion_decode.c new file mode 100644 index 00000000000..c84262c0d50 --- /dev/null +++ b/contrib/density/src/algorithms/lion/core/lion_decode.c @@ -0,0 +1,327 @@ +/* + * Density + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 24/06/15 20:55 + * + * -------------- + * Lion algorithm + * -------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Multiform compression algorithm + */ + +#include "lion_decode.h" + +DENSITY_FORCE_INLINE void density_lion_decode_read_signature(const uint8_t **DENSITY_RESTRICT in, uint_fast64_t *const DENSITY_RESTRICT signature) { +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(signature, *in, sizeof(density_lion_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + density_lion_signature endian_signature; + DENSITY_MEMCPY(&endian_signature, *in, sizeof(density_lion_signature)); + *signature = DENSITY_LITTLE_ENDIAN_64(endian_signature); +#else +#error +#endif + *in += sizeof(density_lion_signature); +} + +DENSITY_FORCE_INLINE void density_lion_decode_update_predictions_model(density_lion_dictionary_chunk_prediction_entry *const DENSITY_RESTRICT predictions, const uint32_t chunk) { + DENSITY_MEMMOVE((uint32_t *) predictions + 1, predictions, 2 * sizeof(uint32_t)); + *(uint32_t *) predictions = chunk; // Move chunk to the top of the predictions list, does not ensure dictionary content consistency between endiannesses +} + +DENSITY_FORCE_INLINE void density_lion_decode_update_dictionary_model(density_lion_dictionary_chunk_entry *const DENSITY_RESTRICT entry, const uint32_t chunk) { + DENSITY_MEMMOVE((uint32_t *) entry + 1, entry, 3 * sizeof(uint32_t)); + *(uint32_t *) entry = chunk; // Does not ensure dictionary content consistency between endiannesses +} + +DENSITY_FORCE_INLINE void density_lion_decode_read_hash(const uint8_t **DENSITY_RESTRICT in, uint16_t *DENSITY_RESTRICT const hash) { +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(hash, *in, sizeof(uint16_t)); +#elif defined(DENSITY_BIG_ENDIAN) + uint16_t endian_hash; + DENSITY_MEMCPY(&endian_hash, *in, sizeof(uint16_t)); + *hash = DENSITY_LITTLE_ENDIAN_16(endian_hash); +#else +#error +#endif + *in += sizeof(uint16_t); +} + +DENSITY_FORCE_INLINE void density_lion_decode_prediction_generic(uint8_t **DENSITY_RESTRICT out, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { + *hash = DENSITY_LION_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)); + DENSITY_MEMCPY(*out, unit, sizeof(uint32_t)); + *out += sizeof(uint32_t); +} + +DENSITY_FORCE_INLINE void density_lion_decode_dictionary_generic(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT const unit) { + DENSITY_MEMCPY(*out, unit, sizeof(uint32_t)); + *out += sizeof(uint32_t); + density_lion_dictionary_chunk_prediction_entry *prediction = &(dictionary->predictions[*last_hash]); + density_lion_decode_update_predictions_model(prediction, *unit); +} + +void density_lion_decode_prediction_a(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { + (void)in; + *unit = dictionary->predictions[*last_hash].next_chunk_a; + density_lion_decode_prediction_generic(out, hash, unit); + + *last_hash = *hash; +} + +void density_lion_decode_prediction_b(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { + (void)in; + density_lion_dictionary_chunk_prediction_entry *const prediction = &dictionary->predictions[*last_hash]; + *unit = prediction->next_chunk_b; + density_lion_decode_update_predictions_model(prediction, *unit); + density_lion_decode_prediction_generic(out, hash, unit); + + *last_hash = *hash; +} + +void density_lion_decode_prediction_c(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { + (void)in; + density_lion_dictionary_chunk_prediction_entry *const prediction = &dictionary->predictions[*last_hash]; + *unit = prediction->next_chunk_c; + density_lion_decode_update_predictions_model(prediction, *unit); + density_lion_decode_prediction_generic(out, hash, unit); + + *last_hash = *hash; +} + +void density_lion_decode_dictionary_a(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { + density_lion_decode_read_hash(in, hash); + DENSITY_PREFETCH(&dictionary->predictions[*hash]); + *unit = dictionary->chunks[*hash].chunk_a; + density_lion_decode_dictionary_generic(out, last_hash, dictionary, unit); + + *last_hash = *hash; +} + +void density_lion_decode_dictionary_b(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { + density_lion_decode_read_hash(in, hash); + DENSITY_PREFETCH(&dictionary->predictions[*hash]); + density_lion_dictionary_chunk_entry *entry = &dictionary->chunks[*hash]; + *unit = entry->chunk_b; + density_lion_decode_update_dictionary_model(entry, *unit); + density_lion_decode_dictionary_generic(out, last_hash, dictionary, unit); + + *last_hash = *hash; +} + +void density_lion_decode_dictionary_c(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { + density_lion_decode_read_hash(in, hash); + DENSITY_PREFETCH(&dictionary->predictions[*hash]); + density_lion_dictionary_chunk_entry *entry = &dictionary->chunks[*hash]; + *unit = entry->chunk_c; + density_lion_decode_update_dictionary_model(entry, *unit); + density_lion_decode_dictionary_generic(out, last_hash, dictionary, unit); + + *last_hash = *hash; +} + +void density_lion_decode_dictionary_d(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { + density_lion_decode_read_hash(in, hash); + DENSITY_PREFETCH(&dictionary->predictions[*hash]); + density_lion_dictionary_chunk_entry *entry = &dictionary->chunks[*hash]; + *unit = entry->chunk_d; + density_lion_decode_update_dictionary_model(entry, *unit); + density_lion_decode_dictionary_generic(out, last_hash, dictionary, unit); + + *last_hash = *hash; +} + +void density_lion_decode_plain(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { + DENSITY_MEMCPY(unit, *in, sizeof(uint32_t)); + *in += sizeof(uint32_t); + *hash = DENSITY_LION_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)); + density_lion_dictionary_chunk_entry *entry = &dictionary->chunks[*hash]; + density_lion_decode_update_dictionary_model(entry, *unit); + DENSITY_MEMCPY(*out, unit, sizeof(uint32_t)); + *out += sizeof(uint32_t); + density_lion_dictionary_chunk_prediction_entry *prediction = &(dictionary->predictions[*last_hash]); + density_lion_decode_update_predictions_model(prediction, *unit); + + *last_hash = *hash; +} + +DENSITY_FORCE_INLINE void density_lion_decode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const data, const DENSITY_LION_FORM form) { + uint16_t hash; + uint32_t unit; + + data->attachments[form](in, out, last_hash, dictionary, &hash, &unit); +} + +DENSITY_FORCE_INLINE DENSITY_LION_FORM density_lion_decode_read_form(const uint8_t **DENSITY_RESTRICT in, uint_fast64_t *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_form_data *const form_data) { + const uint_fast8_t trailing_zeroes = DENSITY_CTZ(0x80 | (*signature >> *shift)); + if (DENSITY_LIKELY(!trailing_zeroes)) { + *shift = (uint_fast8_t)((*shift + 1) & 0x3f); + return density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool); + } else if (DENSITY_LIKELY(trailing_zeroes <= 6)) { + *shift = (uint_fast8_t)((*shift + (trailing_zeroes + 1)) & 0x3f); + return density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool + trailing_zeroes); + } else { + if (DENSITY_LIKELY(*shift <= (density_bitsizeof(density_lion_signature) - 7))) { + *shift = (uint_fast8_t)((*shift + 7) & 0x3f); + return density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool + 7); + } else { + density_lion_decode_read_signature(in, signature); + const uint_fast8_t primary_trailing_zeroes = (uint_fast8_t)(density_bitsizeof(density_lion_signature) - *shift); + const uint_fast8_t ctz_barrier_shift = (uint_fast8_t)(7 - primary_trailing_zeroes); + const uint_fast8_t secondary_trailing_zeroes = DENSITY_CTZ(((uint64_t)1 << ctz_barrier_shift) | *signature); + if (DENSITY_LIKELY(secondary_trailing_zeroes != ctz_barrier_shift)) + *shift = (uint_fast8_t)(secondary_trailing_zeroes + 1); + else + *shift = secondary_trailing_zeroes; + return density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool + primary_trailing_zeroes + secondary_trailing_zeroes); + } + } +} + +DENSITY_FORCE_INLINE void density_lion_decode_process_form(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const form_data, uint_fast64_t *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift) { + if (DENSITY_UNLIKELY(!*shift)) + density_lion_decode_read_signature(in, signature); + + switch ((*signature >> *shift) & 0x1) { + case 0: + density_lion_decode_4(in, out, last_hash, dictionary, form_data, density_lion_decode_read_form(in, signature, shift, form_data)); + break; + default: + density_lion_decode_4(in, out, last_hash, dictionary, form_data, density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool)); + *shift = (uint_fast8_t)((*shift + 1) & 0x3f); + break; + } +} + +DENSITY_FORCE_INLINE void density_lion_decode_256(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const form_data, uint_fast64_t *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift) { +#ifdef __clang__ + for (uint_fast8_t count = 0; count < (DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG >> 2); count++) { + DENSITY_UNROLL_4(density_lion_decode_process_form(in, out, last_hash, dictionary, form_data, signature, shift)); + } +#else + for (uint_fast8_t count = 0; count < (DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG >> 2); count++) { + DENSITY_UNROLL_4(density_lion_decode_process_form(in, out, last_hash, dictionary, form_data, signature, shift)); + } +#endif +} + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_lion_decode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) { + if (out_size < DENSITY_LION_MAXIMUM_DECOMPRESSED_UNIT_SIZE) + return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; + + density_lion_signature signature = 0; + density_lion_form_data data; + density_lion_form_model_init(&data); + void (*attachments[DENSITY_LION_NUMBER_OF_FORMS])(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const) = {(void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_prediction_a, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_prediction_b, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_prediction_c, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_dictionary_a, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_dictionary_b, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_dictionary_c, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_dictionary_d, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_plain}; + density_lion_form_model_attach(&data, attachments); + uint_fast8_t shift = 0; + uint_fast64_t remaining; + uint_fast16_t last_hash = 0; + DENSITY_LION_FORM form; + + const uint8_t *start = *in; + + if (in_size < DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE) { + goto read_and_decode_4; + } + + const uint8_t *in_limit = *in + in_size - DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE; + uint8_t *out_limit = *out + out_size - DENSITY_LION_MAXIMUM_DECOMPRESSED_UNIT_SIZE; + + while (DENSITY_LIKELY(*in <= in_limit && *out <= out_limit)) { + if (DENSITY_UNLIKELY(!(state->counter & 0xf))) { + DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START; + } + state->counter++; + if (DENSITY_UNLIKELY(state->copy_penalty)) { + DENSITY_ALGORITHM_COPY(DENSITY_LION_WORK_BLOCK_SIZE); + DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START; + } else { + const uint8_t *in_start = *in; + density_lion_decode_256(in, out, &last_hash, (density_lion_dictionary *const) state->dictionary, &data, &signature, &shift); + DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*in - in_start), DENSITY_LION_WORK_BLOCK_SIZE); + } + } + + if (*out > out_limit) + return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; + + read_and_decode_4: + if (DENSITY_UNLIKELY(!shift)) { + if (in_size - (*in - start) < sizeof(density_lion_signature)) + return DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL; + + density_lion_decode_read_signature(in, &signature); + } + form = density_lion_decode_read_form(in, &signature, &shift, &data); + switch (in_size - (*in - start)) { + case 0: + case 1: + switch (form) { + case DENSITY_LION_FORM_PLAIN: + goto process_remaining_bytes; // End marker + case DENSITY_LION_FORM_PREDICTIONS_A: + case DENSITY_LION_FORM_PREDICTIONS_B: + case DENSITY_LION_FORM_PREDICTIONS_C: + density_lion_decode_4(in, out, &last_hash, (density_lion_dictionary *const) state->dictionary, &data, form); + break; + default: + return DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING; // Not enough bytes to read a hash + } + break; + case 2: + case 3: + switch (form) { + case DENSITY_LION_FORM_PLAIN: + goto process_remaining_bytes; // End marker + default: + density_lion_decode_4(in, out, &last_hash, (density_lion_dictionary *const) state->dictionary, &data, form); + break; + } + break; + default: + density_lion_decode_4(in, out, &last_hash, (density_lion_dictionary *const) state->dictionary, &data, form); + break; + } + goto read_and_decode_4; + + process_remaining_bytes: + remaining = in_size - (*in - start); + DENSITY_MEMCPY(*out, *in, remaining); + *in += remaining; + *out += remaining; + + return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED; +} diff --git a/contrib/density/src/algorithms/lion/core/lion_decode.h b/contrib/density/src/algorithms/lion/core/lion_decode.h new file mode 100644 index 00000000000..9972f0be388 --- /dev/null +++ b/contrib/density/src/algorithms/lion/core/lion_decode.h @@ -0,0 +1,54 @@ +/* + * Density + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 24/06/15 20:55 + * + * -------------- + * Lion algorithm + * -------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Multiform compression algorithm + */ + +#ifndef DENSITY_LION_DECODE_H +#define DENSITY_LION_DECODE_H + +#include "../dictionary/lion_dictionary.h" +#include "../forms/lion_form_model.h" +#include "../../algorithms.h" + +DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_lion_decode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t); + +#endif diff --git a/contrib/density/src/algorithms/lion/core/lion_encode.c b/contrib/density/src/algorithms/lion/core/lion_encode.c new file mode 100644 index 00000000000..7cff2f3dda7 --- /dev/null +++ b/contrib/density/src/algorithms/lion/core/lion_encode.c @@ -0,0 +1,298 @@ +/* + * Density + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 24/06/15 18:57 + * + * -------------- + * Lion algorithm + * -------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Multiform compression algorithm + */ + +#include "lion_encode.h" + +DENSITY_FORCE_INLINE void density_lion_encode_prepare_signature(uint8_t **DENSITY_RESTRICT out, uint_fast64_t **DENSITY_RESTRICT signature_pointer, uint_fast64_t *const DENSITY_RESTRICT signature) { + *signature = 0; + *signature_pointer = (density_lion_signature *) *out; + *out += sizeof(density_lion_signature); +} + +DENSITY_FORCE_INLINE void density_lion_encode_push_to_proximity_signature(uint_fast64_t *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, const uint64_t content, const uint_fast8_t bits) { + *signature |= (content << *shift); + *shift += bits; +} + +DENSITY_FORCE_INLINE void density_lion_encode_push_to_signature(uint8_t **DENSITY_RESTRICT out, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, const uint64_t content, const uint_fast8_t bits) { + if (DENSITY_LIKELY(*shift)) { + density_lion_encode_push_to_proximity_signature(signature, shift, content, bits); + + if (DENSITY_UNLIKELY(*shift >= density_bitsizeof(density_lion_signature))) { +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(*signature_pointer, signature, sizeof(density_lion_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + const density_lion_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(*signature); + DENSITY_MEMCPY(*signature_pointer, &endian_signature, sizeof(density_lion_signature)); +#else +#error +#endif + + const uint_fast8_t remainder = (uint_fast8_t)(*shift & 0x3f); + *shift = 0; + if (remainder) { + density_lion_encode_prepare_signature(out, signature_pointer, signature); + density_lion_encode_push_to_proximity_signature(signature, shift, content >> (bits - remainder), remainder); + } + } + } else { + density_lion_encode_prepare_signature(out, signature_pointer, signature); + density_lion_encode_push_to_proximity_signature(signature, shift, content, bits); + } +} + +DENSITY_FORCE_INLINE void density_lion_encode_push_zero_to_signature(uint8_t **DENSITY_RESTRICT out, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, const uint_fast8_t bits) { + if (DENSITY_LIKELY(*shift)) { + *shift += bits; + + if (DENSITY_UNLIKELY(*shift >= density_bitsizeof(density_lion_signature))) { +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(*signature_pointer, signature, sizeof(density_lion_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + const density_lion_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(*signature); + DENSITY_MEMCPY(*signature_pointer, &endian_signature, sizeof(density_lion_signature)); +#else +#error +#endif + + const uint_fast8_t remainder = (uint_fast8_t)(*shift & 0x3f); + if (remainder) { + density_lion_encode_prepare_signature(out, signature_pointer, signature); + *shift = remainder; + } else + *shift = 0; + } + } else { + density_lion_encode_prepare_signature(out, signature_pointer, signature); + *shift = bits; + } +} + +DENSITY_FORCE_INLINE void density_lion_encode_push_code_to_signature(uint8_t **DENSITY_RESTRICT out, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, const density_lion_entropy_code code) { + density_lion_encode_push_to_signature(out, signature_pointer, signature, shift, code.value, code.bitLength); +} + +DENSITY_FORCE_INLINE void density_lion_encode_kernel_4(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, const uint16_t hash, density_lion_form_data *const data, const uint32_t unit) { + density_lion_dictionary_chunk_prediction_entry *const predictions = &dictionary->predictions[*last_hash]; + DENSITY_PREFETCH(&dictionary->predictions[hash]); + + if (*(uint32_t *) predictions ^ unit) { + if (*((uint32_t *) predictions + 1) ^ unit) { + if (*((uint32_t *) predictions + 2) ^ unit) { + density_lion_dictionary_chunk_entry *const in_dictionary = &dictionary->chunks[hash]; + if (*(uint32_t *) in_dictionary ^ unit) { + if (*((uint32_t *) in_dictionary + 1) ^ unit) { + if (*((uint32_t *) in_dictionary + 2) ^ unit) { + if (*((uint32_t *) in_dictionary + 3) ^ unit) { + density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_PLAIN)); + DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t)); + *out += sizeof(uint32_t); + } else { + density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_DICTIONARY_D)); +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); +#elif defined(DENSITY_BIG_ENDIAN) + const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); + DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); +#else +#error +#endif + *out += sizeof(uint16_t); + } + } else { + density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_DICTIONARY_C)); +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); +#elif defined(DENSITY_BIG_ENDIAN) + const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); + DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); +#else +#error +#endif + *out += sizeof(uint16_t); + } + } else { + density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_DICTIONARY_B)); +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); +#elif defined(DENSITY_BIG_ENDIAN) + const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); + DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); +#else +#error +#endif + *out += sizeof(uint16_t); + } + DENSITY_MEMMOVE((uint32_t *) in_dictionary + 1, in_dictionary, 3 * sizeof(uint32_t)); + *(uint32_t *) in_dictionary = unit; // Does not ensure dictionary content consistency between endiannesses + } else { + density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_DICTIONARY_A)); +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); +#elif defined(DENSITY_BIG_ENDIAN) + const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); + DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); +#else +#error +#endif + *out += sizeof(uint16_t); + } + } else { + density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_PREDICTIONS_C)); + } + } else { + density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_PREDICTIONS_B)); + } + DENSITY_MEMMOVE((uint32_t *) predictions + 1, predictions, 2 * sizeof(uint32_t)); + *(uint32_t *) predictions = unit; // Does not ensure dictionary content consistency between endiannesses + } else + density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_PREDICTIONS_A)); + *last_hash = hash; +} + +DENSITY_FORCE_INLINE void density_lion_encode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const data, uint32_t *DENSITY_RESTRICT unit) { + DENSITY_MEMCPY(unit, *in, sizeof(uint32_t)); + density_lion_encode_kernel_4(out, last_hash, signature_pointer, signature, shift, dictionary, DENSITY_LION_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)), data, *unit); + *in += sizeof(uint32_t); +} + +DENSITY_FORCE_INLINE void density_lion_encode_generic(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, const uint_fast8_t chunks_per_process_unit, density_lion_form_data *const data, uint32_t *DENSITY_RESTRICT unit) { +#ifdef __clang__ + for (uint_fast8_t count = 0; count < (chunks_per_process_unit >> 2); count++) { + DENSITY_UNROLL_4(density_lion_encode_4(in, out, last_hash, signature_pointer, signature, shift, dictionary, data, unit)); + } +#else + for (uint_fast8_t count = 0; count < (chunks_per_process_unit >> 1); count++) { + DENSITY_UNROLL_2(density_lion_encode_4(in, out, last_hash, signature_pointer, signature, shift, dictionary, data, unit)); + } +#endif +} + +DENSITY_FORCE_INLINE void density_lion_encode_32(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const data, uint32_t *DENSITY_RESTRICT unit) { + density_lion_encode_generic(in, out, last_hash, signature_pointer, signature, shift, dictionary, DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_SMALL, data, unit); +} + +DENSITY_FORCE_INLINE void density_lion_encode_256(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const data, uint32_t *DENSITY_RESTRICT unit) { + density_lion_encode_generic(in, out, last_hash, signature_pointer, signature, shift, dictionary, DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG, data, unit); +} + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_lion_encode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) { + if (out_size < DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE) + return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; + + density_lion_signature signature = 0; + density_lion_signature *signature_pointer = NULL; + uint_fast8_t shift = 0; + density_lion_form_data data; + density_lion_form_model_init(&data); + uint_fast16_t last_hash = 0; + uint32_t unit; + + uint8_t *out_limit = *out + out_size - DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE; + uint_fast64_t limit_256 = (in_size >> 8); + + while (DENSITY_LIKELY(limit_256-- && *out <= out_limit)) { + if (DENSITY_UNLIKELY(!(state->counter & 0xf))) { + DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START; + } + state->counter++; + if (DENSITY_UNLIKELY(state->copy_penalty)) { + DENSITY_ALGORITHM_COPY(DENSITY_LION_WORK_BLOCK_SIZE); + DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START; + } else { + const uint8_t *out_start = *out; + DENSITY_PREFETCH(*in + DENSITY_LION_WORK_BLOCK_SIZE); + density_lion_encode_256(in, out, &last_hash, &signature_pointer, &signature, &shift, (density_lion_dictionary *const) state->dictionary, &data, &unit); + DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*out - out_start), DENSITY_LION_WORK_BLOCK_SIZE); + } + } + + if (*out > out_limit) + return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; + + uint_fast64_t remaining; + + switch (in_size & 0xff) { + case 0: + case 1: + case 2: + case 3: + density_lion_encode_push_code_to_signature(out, &signature_pointer, &signature, &shift, density_lion_form_model_get_encoding(&data, DENSITY_LION_FORM_PLAIN)); // End marker +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_lion_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + const density_lion_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); + DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_lion_signature)); +#else +#error +#endif + goto process_remaining_bytes; + default: + break; + } + + uint_fast64_t limit_4 = (in_size & 0xff) >> 2; + while (limit_4--) + density_lion_encode_4(in, out, &last_hash, &signature_pointer, &signature, &shift, (density_lion_dictionary *const) state->dictionary, &data, &unit); + + density_lion_encode_push_code_to_signature(out, &signature_pointer, &signature, &shift, density_lion_form_model_get_encoding(&data, DENSITY_LION_FORM_PLAIN)); // End marker +#ifdef DENSITY_LITTLE_ENDIAN + DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_lion_signature)); +#elif defined(DENSITY_BIG_ENDIAN) + const density_lion_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); + DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_lion_signature)); +#else +#error +#endif + + process_remaining_bytes: + remaining = in_size & 0x3; + if (remaining) { + DENSITY_MEMCPY(*out, *in, remaining); + *in += remaining; + *out += remaining; + } + + return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED; +} diff --git a/contrib/density/src/algorithms/lion/core/lion_encode.h b/contrib/density/src/algorithms/lion/core/lion_encode.h new file mode 100644 index 00000000000..71cd2cad4cf --- /dev/null +++ b/contrib/density/src/algorithms/lion/core/lion_encode.h @@ -0,0 +1,54 @@ +/* + * Density + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 24/06/15 18:56 + * + * -------------- + * Lion algorithm + * -------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Multiform compression algorithm + */ + +#ifndef DENSITY_LION_ENCODE_H +#define DENSITY_LION_ENCODE_H + +#include "../dictionary/lion_dictionary.h" +#include "../forms/lion_form_model.h" +#include "../../algorithms.h" + +DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_lion_encode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t); + +#endif diff --git a/contrib/density/src/algorithms/lion/dictionary/lion_dictionary.h b/contrib/density/src/algorithms/lion/dictionary/lion_dictionary.h new file mode 100644 index 00000000000..68f71da6373 --- /dev/null +++ b/contrib/density/src/algorithms/lion/dictionary/lion_dictionary.h @@ -0,0 +1,73 @@ +/* + * Density + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 12/02/15 23:09 + * + * -------------- + * Lion algorithm + * -------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Multiform compression algorithm + */ + +#ifndef DENSITY_LION_DICTIONARY_H +#define DENSITY_LION_DICTIONARY_H + +#include "../lion.h" + +#pragma pack(push) +#pragma pack(4) + +typedef struct { + uint32_t chunk_a; + uint32_t chunk_b; + uint32_t chunk_c; + uint32_t chunk_d; + uint32_t chunk_e; +} density_lion_dictionary_chunk_entry; + +typedef struct { + uint32_t next_chunk_a; + uint32_t next_chunk_b; + uint32_t next_chunk_c; +} density_lion_dictionary_chunk_prediction_entry; + +typedef struct { + density_lion_dictionary_chunk_entry chunks[1 << DENSITY_LION_CHUNK_HASH_BITS]; + density_lion_dictionary_chunk_prediction_entry predictions[1 << DENSITY_LION_CHUNK_HASH_BITS]; +} density_lion_dictionary; +#pragma pack(pop) + +#endif diff --git a/contrib/density/src/algorithms/lion/forms/lion_form_model.c b/contrib/density/src/algorithms/lion/forms/lion_form_model.c new file mode 100644 index 00000000000..21dfc23e3c9 --- /dev/null +++ b/contrib/density/src/algorithms/lion/forms/lion_form_model.c @@ -0,0 +1,153 @@ +/* + * Density + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 9/03/15 11:19 + * + * -------------- + * Lion algorithm + * -------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Multiform compression algorithm + */ + +#include "lion_form_model.h" + +const density_lion_entropy_code density_lion_form_entropy_codes[DENSITY_LION_NUMBER_OF_FORMS] = DENSITY_LION_FORM_MODEL_ENTROPY_CODES; + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_lion_form_model_init(density_lion_form_data *const data) { + density_lion_form_node *rank_0 = &data->formsPool[0]; + rank_0->form = DENSITY_LION_FORM_PLAIN; + rank_0->rank = 0; + rank_0->previousForm = NULL; + data->formsIndex[DENSITY_LION_FORM_PLAIN] = rank_0; + + density_lion_form_node *rank_1 = &data->formsPool[1]; + rank_1->form = DENSITY_LION_FORM_DICTIONARY_A; + rank_1->rank = 1; + rank_1->previousForm = rank_0; + data->formsIndex[DENSITY_LION_FORM_DICTIONARY_A] = rank_1; + + density_lion_form_node *rank_2 = &data->formsPool[2]; + rank_2->form = DENSITY_LION_FORM_DICTIONARY_B; + rank_2->rank = 2; + rank_2->previousForm = rank_1; + data->formsIndex[DENSITY_LION_FORM_DICTIONARY_B] = rank_2; + + density_lion_form_node *rank_3 = &data->formsPool[3]; + rank_3->form = DENSITY_LION_FORM_PREDICTIONS_A; + rank_3->rank = 3; + rank_3->previousForm = rank_2; + data->formsIndex[DENSITY_LION_FORM_PREDICTIONS_A] = rank_3; + + density_lion_form_node *rank_4 = &data->formsPool[4]; + rank_4->form = DENSITY_LION_FORM_PREDICTIONS_B; + rank_4->rank = 4; + rank_4->previousForm = rank_3; + data->formsIndex[DENSITY_LION_FORM_PREDICTIONS_B] = rank_4; + + density_lion_form_node *rank_5 = &data->formsPool[5]; + rank_5->form = DENSITY_LION_FORM_DICTIONARY_C; + rank_5->rank = 5; + rank_5->previousForm = rank_4; + data->formsIndex[DENSITY_LION_FORM_DICTIONARY_C] = rank_5; + + density_lion_form_node *rank_6 = &data->formsPool[6]; + rank_6->form = DENSITY_LION_FORM_PREDICTIONS_C; + rank_6->rank = 6; + rank_6->previousForm = rank_5; + data->formsIndex[DENSITY_LION_FORM_PREDICTIONS_C] = rank_6; + + density_lion_form_node *rank_7 = &data->formsPool[7]; + rank_7->form = DENSITY_LION_FORM_DICTIONARY_D; + rank_7->rank = 7; + rank_7->previousForm = rank_6; + data->formsIndex[DENSITY_LION_FORM_DICTIONARY_D] = rank_7; + + data->usages.usages_as_uint64_t = 0; +} + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_lion_form_model_attach(density_lion_form_data *const data, void (*attachments[DENSITY_LION_NUMBER_OF_FORMS])(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) { + for(uint_fast8_t count = 0; count < DENSITY_LION_NUMBER_OF_FORMS; count ++) + data->attachments[count] = attachments[count]; +} + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_lion_form_model_update(density_lion_form_data *const DENSITY_RESTRICT data, density_lion_form_node *const DENSITY_RESTRICT form, const uint8_t usage, density_lion_form_node *const DENSITY_RESTRICT previous_form, const uint8_t previous_usage) { + if (DENSITY_UNLIKELY(previous_usage < usage)) { // Relative stability is assumed + const DENSITY_LION_FORM form_value = form->form; + const DENSITY_LION_FORM previous_form_value = previous_form->form; + + previous_form->form = form_value; + form->form = previous_form_value; + + data->formsIndex[form_value] = previous_form; + data->formsIndex[previous_form_value] = form; + } +} + +DENSITY_FORCE_INLINE void density_lion_form_model_flatten(density_lion_form_data *const data, const uint8_t usage) { + if (DENSITY_UNLIKELY(usage & 0x80)) + data->usages.usages_as_uint64_t = (data->usages.usages_as_uint64_t >> 1) & 0x7f7f7f7f7f7f7f7fllu; // Flatten usage values +} + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE DENSITY_LION_FORM density_lion_form_model_increment_usage(density_lion_form_data *const data, density_lion_form_node *const DENSITY_RESTRICT form) { + const DENSITY_LION_FORM form_value = form->form; + const uint8_t usage = ++data->usages.usages_as_uint8_t[form_value]; + + density_lion_form_node *const previous_form = form->previousForm; + + if (previous_form) + density_lion_form_model_update(data, form, usage, previous_form, data->usages.usages_as_uint8_t[previous_form->form]); + else + density_lion_form_model_flatten(data, usage); + + return form_value; +} + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_lion_entropy_code density_lion_form_model_get_encoding(density_lion_form_data *const data, const DENSITY_LION_FORM form) { + const uint8_t usage = ++data->usages.usages_as_uint8_t[form]; + + density_lion_form_node *const form_found = data->formsIndex[form]; + density_lion_form_node *const previous_form = form_found->previousForm; + + if (previous_form) { + density_lion_form_model_update(data, form_found, usage, previous_form, data->usages.usages_as_uint8_t[previous_form->form]); + + return density_lion_form_entropy_codes[form_found->rank]; + } else { + density_lion_form_model_flatten(data, usage); + + return density_lion_form_entropy_codes[0]; + } +} diff --git a/contrib/density/src/algorithms/lion/forms/lion_form_model.h b/contrib/density/src/algorithms/lion/forms/lion_form_model.h new file mode 100644 index 00000000000..90164d94c78 --- /dev/null +++ b/contrib/density/src/algorithms/lion/forms/lion_form_model.h @@ -0,0 +1,96 @@ +/* + * Density + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 9/03/15 12:04 + * + * -------------- + * Lion algorithm + * -------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Multiform compression algorithm + */ + +#ifndef DENSITY_LION_FORM_MODEL_H +#define DENSITY_LION_FORM_MODEL_H + +#include "../../../globals.h" +#include "../lion.h" + +#define DENSITY_LION_NUMBER_OF_FORMS 8 + +// Unary codes (reversed) except the last one +#define DENSITY_LION_FORM_MODEL_ENTROPY_CODES {\ + {DENSITY_BINARY_TO_UINT(1), 1},\ + {DENSITY_BINARY_TO_UINT(10), 2},\ + {DENSITY_BINARY_TO_UINT(100), 3},\ + {DENSITY_BINARY_TO_UINT(1000), 4},\ + {DENSITY_BINARY_TO_UINT(10000), 5},\ + {DENSITY_BINARY_TO_UINT(100000), 6},\ + {DENSITY_BINARY_TO_UINT(1000000), 7},\ + {DENSITY_BINARY_TO_UINT(0000000), 7},\ +} + +#pragma pack(push) +#pragma pack(4) +typedef struct { + void* previousForm; + DENSITY_LION_FORM form; + uint8_t rank; +} density_lion_form_node; + +typedef struct { + union { + uint8_t usages_as_uint8_t[DENSITY_LION_NUMBER_OF_FORMS]; + uint64_t usages_as_uint64_t; + } usages; + + void (*attachments[DENSITY_LION_NUMBER_OF_FORMS])(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const); + density_lion_form_node formsPool[DENSITY_LION_NUMBER_OF_FORMS]; + density_lion_form_node *formsIndex[DENSITY_LION_NUMBER_OF_FORMS]; + uint8_t nextAvailableForm; +} density_lion_form_data; +#pragma pack(pop) + +DENSITY_WINDOWS_EXPORT void density_lion_form_model_init(density_lion_form_data *const); + +DENSITY_WINDOWS_EXPORT void density_lion_form_model_attach(density_lion_form_data *const, void (*[DENSITY_LION_NUMBER_OF_FORMS])(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)); + +DENSITY_WINDOWS_EXPORT void density_lion_form_model_update(density_lion_form_data *const DENSITY_RESTRICT_DECLARE, density_lion_form_node *const DENSITY_RESTRICT_DECLARE, const uint8_t, density_lion_form_node *const DENSITY_RESTRICT_DECLARE, const uint8_t); + +DENSITY_WINDOWS_EXPORT DENSITY_LION_FORM density_lion_form_model_increment_usage(density_lion_form_data *const, density_lion_form_node *const DENSITY_RESTRICT_DECLARE); + +DENSITY_WINDOWS_EXPORT density_lion_entropy_code density_lion_form_model_get_encoding(density_lion_form_data *const, const DENSITY_LION_FORM); + +#endif diff --git a/contrib/density/src/algorithms/lion/lion.h b/contrib/density/src/algorithms/lion/lion.h new file mode 100644 index 00000000000..30b40f8d92b --- /dev/null +++ b/contrib/density/src/algorithms/lion/lion.h @@ -0,0 +1,94 @@ +/* + * Density + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 5/02/15 20:57 + * + * -------------- + * Lion algorithm + * -------------- + * + * Author(s) + * Guillaume Voirin (https://github.com/gpnuma) + * + * Description + * Multiform compression algorithm + */ + +#ifndef DENSITY_LION_H +#define DENSITY_LION_H + +#include "../../globals.h" + +#define DENSITY_LION_HASH32_MULTIPLIER (uint32_t)0x9D6EF916lu +#define DENSITY_LION_CHUNK_HASH_BITS 16 + +#define DENSITY_LION_HASH_ALGORITHM(value32) (uint16_t)(value32 * DENSITY_LION_HASH32_MULTIPLIER >> (32 - DENSITY_LION_CHUNK_HASH_BITS)) + +typedef enum { + DENSITY_LION_FORM_PREDICTIONS_A = 0, + DENSITY_LION_FORM_PREDICTIONS_B, + DENSITY_LION_FORM_PREDICTIONS_C, + DENSITY_LION_FORM_DICTIONARY_A, + DENSITY_LION_FORM_DICTIONARY_B, + DENSITY_LION_FORM_DICTIONARY_C, + DENSITY_LION_FORM_DICTIONARY_D, + DENSITY_LION_FORM_PLAIN, +} DENSITY_LION_FORM; + +typedef enum { + DENSITY_LION_PREDICTIONS_SIGNATURE_FLAG_A = 0x0, + DENSITY_LION_PREDICTIONS_SIGNATURE_FLAG_B = 0x1, +} DENSITY_LION_PREDICTIONS_SIGNATURE_FLAG; + +#pragma pack(push) +#pragma pack(4) +typedef struct { + uint_fast8_t value; + uint_fast8_t bitLength; +} density_lion_entropy_code; +#pragma pack(pop) + +typedef uint64_t density_lion_signature; + +#define DENSITY_LION_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE (density_bitsizeof(density_lion_signature) * sizeof(uint32_t)) // Plain writes +#define DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE (sizeof(density_lion_signature) + DENSITY_LION_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE) + +#define DENSITY_LION_MAXIMUM_DECOMPRESSED_UNIT_SIZE (density_bitsizeof(density_lion_signature) * sizeof(uint32_t)) // Smallest form size times work unit size + +#define DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_SMALL 8 +#define DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG 64 +#define DENSITY_LION_PROCESS_UNIT_SIZE_SMALL (DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_SMALL * sizeof(uint32_t)) +#define DENSITY_LION_PROCESS_UNIT_SIZE_BIG (DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG * sizeof(uint32_t)) + +#define DENSITY_LION_WORK_BLOCK_SIZE 256 +#define DENSITY_LION_COPY_PENALTY 2 + +#endif diff --git a/contrib/density/src/buffers/buffer.c b/contrib/density/src/buffers/buffer.c new file mode 100644 index 00000000000..d3e9943c599 --- /dev/null +++ b/contrib/density/src/buffers/buffer.c @@ -0,0 +1,220 @@ +/* + * Density + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 3/02/15 19:53 + */ + +#include "buffer.h" + +DENSITY_WINDOWS_EXPORT uint_fast64_t density_compress_safe_size(const uint_fast64_t input_size) { + const uint_fast64_t slack = DENSITY_MAX_3(DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE, DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE, DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE); + + // Chameleon longest output + uint_fast64_t chameleon_longest_output_size = 0; + chameleon_longest_output_size += sizeof(density_header); + chameleon_longest_output_size += sizeof(density_chameleon_signature) * (1 + (input_size >> (5 + 3))); // Signature space (1 bit <=> 4 bytes) + chameleon_longest_output_size += sizeof(density_chameleon_signature); // Eventual supplementary signature for end marker + chameleon_longest_output_size += input_size; // Everything encoded as plain data + + // Cheetah longest output + uint_fast64_t cheetah_longest_output_size = 0; + cheetah_longest_output_size += sizeof(density_header); + cheetah_longest_output_size += sizeof(density_cheetah_signature) * (1 + (input_size >> (4 + 3))); // Signature space (2 bits <=> 4 bytes) + cheetah_longest_output_size += sizeof(density_cheetah_signature); // Eventual supplementary signature for end marker + cheetah_longest_output_size += input_size; // Everything encoded as plain data + + // Lion longest output + uint_fast64_t lion_longest_output_size = 0; + lion_longest_output_size += sizeof(density_header); + lion_longest_output_size += sizeof(density_lion_signature) * (1 + ((input_size * 7) >> (5 + 3))); // Signature space (7 bits <=> 4 bytes), although this size is technically impossible + lion_longest_output_size += sizeof(density_lion_signature); // Eventual supplementary signature for end marker + lion_longest_output_size += input_size; // Everything encoded as plain data + + return DENSITY_MAX_3(chameleon_longest_output_size, cheetah_longest_output_size, lion_longest_output_size) + slack; +} + +DENSITY_WINDOWS_EXPORT uint_fast64_t density_decompress_safe_size(const uint_fast64_t expected_decompressed_output_size) { + const uint_fast64_t slack = DENSITY_MAX_3(DENSITY_CHAMELEON_DECOMPRESSED_UNIT_SIZE, DENSITY_CHEETAH_DECOMPRESSED_UNIT_SIZE, DENSITY_LION_MAXIMUM_DECOMPRESSED_UNIT_SIZE); + + return expected_decompressed_output_size + slack; +} + +DENSITY_FORCE_INLINE DENSITY_STATE density_convert_algorithm_exit_status(const density_algorithm_exit_status status) { + switch (status) { + case DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED: + return DENSITY_STATE_OK; + case DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL: + return DENSITY_STATE_ERROR_INPUT_BUFFER_TOO_SMALL; + case DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL: + return DENSITY_STATE_ERROR_OUTPUT_BUFFER_TOO_SMALL; + default: + return DENSITY_STATE_ERROR_DURING_PROCESSING; + } +} + +DENSITY_FORCE_INLINE density_processing_result density_make_result(const DENSITY_STATE state, const uint_fast64_t read, const uint_fast64_t written, density_context *const context) { + density_processing_result result; + result.state = state; + result.bytesRead = read; + result.bytesWritten = written; + result.context = context; + return result; +} + +DENSITY_FORCE_INLINE density_context* density_allocate_context(const DENSITY_ALGORITHM algorithm, const bool custom_dictionary, void *(*mem_alloc)(size_t)) { + density_context* context = mem_alloc(sizeof(density_context)); + context->algorithm = algorithm; + context->dictionary_size = density_get_dictionary_size(context->algorithm); + context->dictionary_type = custom_dictionary; + if(!context->dictionary_type) { + context->dictionary = mem_alloc(context->dictionary_size); + DENSITY_MEMSET(context->dictionary, 0, context->dictionary_size); + } + return context; +} + +DENSITY_WINDOWS_EXPORT void density_free_context(density_context *const context, void (*mem_free)(void *)) { + if(mem_free == NULL) + mem_free = free; + if(!context->dictionary_type) + mem_free(context->dictionary); + mem_free(context); +} + +DENSITY_WINDOWS_EXPORT density_processing_result density_compress_prepare_context(const DENSITY_ALGORITHM algorithm, const bool custom_dictionary, void *(*mem_alloc)(size_t)) { + if(mem_alloc == NULL) + mem_alloc = malloc; + + return density_make_result(DENSITY_STATE_OK, 0, 0, density_allocate_context(algorithm, custom_dictionary, mem_alloc)); +} + +DENSITY_WINDOWS_EXPORT density_processing_result density_compress_with_context(const uint8_t * input_buffer, const uint_fast64_t input_size, uint8_t * output_buffer, const uint_fast64_t output_size, density_context *const context) { + if (output_size < sizeof(density_header)) + return density_make_result(DENSITY_STATE_ERROR_OUTPUT_BUFFER_TOO_SMALL, 0, 0, context); + if(context == NULL) + return density_make_result(DENSITY_STATE_ERROR_INVALID_CONTEXT, 0, 0, context); + + // Variables setup + const uint8_t *in = input_buffer; + uint8_t *out = output_buffer; + density_algorithm_state state; + density_algorithm_exit_status status = DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING; + + // Header + density_header_write(&out, context->algorithm); + + // Compression + density_algorithms_prepare_state(&state, context->dictionary); + switch (context->algorithm) { + case DENSITY_ALGORITHM_CHAMELEON: + status = density_chameleon_encode(&state, &in, input_size, &out, output_size); + break; + case DENSITY_ALGORITHM_CHEETAH: + status = density_cheetah_encode(&state, &in, input_size, &out, output_size); + break; + case DENSITY_ALGORITHM_LION: + status = density_lion_encode(&state, &in, input_size, &out, output_size); + break; + } + + // Result + return density_make_result(density_convert_algorithm_exit_status(status), in - input_buffer, out - output_buffer, context); +} + +DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_prepare_context(const uint8_t *input_buffer, const uint_fast64_t input_size, const bool custom_dictionary, void *(*mem_alloc)(size_t)) { + if (input_size < sizeof(density_header)) + return density_make_result(DENSITY_STATE_ERROR_INPUT_BUFFER_TOO_SMALL, 0, 0, NULL); + + // Variables setup + const uint8_t* in = input_buffer; + if(mem_alloc == NULL) + mem_alloc = malloc; + + // Read header + density_header main_header; + density_header_read(&in, &main_header); + + // Setup context + density_context *const context = density_allocate_context(main_header.algorithm, custom_dictionary, mem_alloc); + return density_make_result(DENSITY_STATE_OK, in - input_buffer, 0, context); +} + +DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_with_context(const uint8_t * input_buffer, const uint_fast64_t input_size, uint8_t * output_buffer, const uint_fast64_t output_size, density_context *const context) { + if(context == NULL) + return density_make_result(DENSITY_STATE_ERROR_INVALID_CONTEXT, 0, 0, context); + + // Variables setup + const uint8_t *in = input_buffer; + uint8_t *out = output_buffer; + density_algorithm_state state; + density_algorithm_exit_status status = DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING; + + // Decompression + density_algorithms_prepare_state(&state, context->dictionary); + switch (context->algorithm) { + case DENSITY_ALGORITHM_CHAMELEON: + status = density_chameleon_decode(&state, &in, input_size, &out, output_size); + break; + case DENSITY_ALGORITHM_CHEETAH: + status = density_cheetah_decode(&state, &in, input_size, &out, output_size); + break; + case DENSITY_ALGORITHM_LION: + status = density_lion_decode(&state, &in, input_size, &out, output_size); + break; + } + + // Result + return density_make_result(density_convert_algorithm_exit_status(status), in - input_buffer, out - output_buffer, context); +} + +DENSITY_WINDOWS_EXPORT density_processing_result density_compress(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size, const DENSITY_ALGORITHM algorithm) { + density_processing_result result = density_compress_prepare_context(algorithm, false, malloc); + if(result.state) { + density_free_context(result.context, free); + return result; + } + + result = density_compress_with_context(input_buffer, input_size, output_buffer, output_size, result.context); + density_free_context(result.context, free); + return result; +} + +DENSITY_WINDOWS_EXPORT density_processing_result density_decompress(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size) { + density_processing_result result = density_decompress_prepare_context(input_buffer, input_size, false, malloc); + if(result.state) { + density_free_context(result.context, free); + return result; + } + + result = density_decompress_with_context(input_buffer + result.bytesRead, input_size - result.bytesRead, output_buffer, output_size, result.context); + density_free_context(result.context, free); + return result; +} diff --git a/contrib/density/src/buffers/buffer.h b/contrib/density/src/buffers/buffer.h new file mode 100644 index 00000000000..f34ab5458d2 --- /dev/null +++ b/contrib/density/src/buffers/buffer.h @@ -0,0 +1,58 @@ +/* + * Density + * + * Copyright (c) 2015, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 3/02/15 19:51 + */ + +#ifndef DENSITY_BUFFER_H +#define DENSITY_BUFFER_H + +#include "../globals.h" +#include "../density_api.h" +#include "../structure/header.h" +#include "../algorithms/chameleon/core/chameleon_encode.h" +#include "../algorithms/chameleon/core/chameleon_decode.h" +#include "../algorithms/cheetah/core/cheetah_encode.h" +#include "../algorithms/cheetah/core/cheetah_decode.h" +#include "../algorithms/lion/core/lion_encode.h" +#include "../algorithms/lion/core/lion_decode.h" + +DENSITY_WINDOWS_EXPORT uint_fast64_t density_compress_safe_size(const uint_fast64_t); +DENSITY_WINDOWS_EXPORT uint_fast64_t density_decompress_safe_size(const uint_fast64_t); +DENSITY_WINDOWS_EXPORT void density_free_context(density_context *const, void (*)(void *)); +DENSITY_WINDOWS_EXPORT density_processing_result density_compress_prepare_context(const DENSITY_ALGORITHM, const bool, void *(*)(size_t)); +DENSITY_WINDOWS_EXPORT density_processing_result density_compress_with_context(const uint8_t *, const uint_fast64_t, uint8_t *, const uint_fast64_t, density_context *const); +DENSITY_WINDOWS_EXPORT density_processing_result density_compress(const uint8_t *, const uint_fast64_t, uint8_t *, const uint_fast64_t, const DENSITY_ALGORITHM); +DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_prepare_context(const uint8_t *, const uint_fast64_t, const bool, void *(*)(size_t)); +DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_with_context(const uint8_t *, const uint_fast64_t, uint8_t *, const uint_fast64_t, density_context *const); +DENSITY_WINDOWS_EXPORT density_processing_result density_decompress(const uint8_t *, const uint_fast64_t, uint8_t *, const uint_fast64_t); + +#endif diff --git a/contrib/density/src/density_api.h b/contrib/density/src/density_api.h new file mode 100644 index 00000000000..385e24e609a --- /dev/null +++ b/contrib/density/src/density_api.h @@ -0,0 +1,220 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 18/10/13 22:41 + */ + +#ifndef DENSITY_API_H +#define DENSITY_API_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#if defined(_WIN64) || defined(_WIN32) +#define DENSITY_WINDOWS_EXPORT __declspec(dllexport) +#else +#define DENSITY_WINDOWS_EXPORT +#endif + + +/*********************************************************************************************************************** + * * + * API data structures * + * * + ***********************************************************************************************************************/ + +typedef uint8_t density_byte; +typedef bool density_bool; + +typedef enum { + DENSITY_ALGORITHM_CHAMELEON = 1, + DENSITY_ALGORITHM_CHEETAH = 2, + DENSITY_ALGORITHM_LION = 3, +} DENSITY_ALGORITHM; + +typedef enum { + DENSITY_STATE_OK = 0, // Everything went alright + DENSITY_STATE_ERROR_INPUT_BUFFER_TOO_SMALL, // Input buffer size is too small + DENSITY_STATE_ERROR_OUTPUT_BUFFER_TOO_SMALL, // Output buffer size is too small + DENSITY_STATE_ERROR_DURING_PROCESSING, // Error during processing + DENSITY_STATE_ERROR_INVALID_CONTEXT, // Invalid context + DENSITY_STATE_ERROR_INVALID_ALGORITHM, // Invalid algorithm +} DENSITY_STATE; + +typedef struct { + DENSITY_ALGORITHM algorithm; + bool dictionary_type; + size_t dictionary_size; + void* dictionary; +} density_context; + +typedef struct { + DENSITY_STATE state; + uint_fast64_t bytesRead; + uint_fast64_t bytesWritten; + density_context* context; +} density_processing_result; + + + +/*********************************************************************************************************************** + * * + * Density version information * + * * + ***********************************************************************************************************************/ + +/* + * Returns the major density version + */ +DENSITY_WINDOWS_EXPORT uint8_t density_version_major(void); + +/* + * Returns the minor density version + */ +DENSITY_WINDOWS_EXPORT uint8_t density_version_minor(void); + +/* + * Returns the density revision + */ +DENSITY_WINDOWS_EXPORT uint8_t density_version_revision(void); + + + +/*********************************************************************************************************************** + * * + * Density API functions * + * * + ***********************************************************************************************************************/ + +/* + * Return the required size of an algorithm's dictionary + * + * @param algorithm the algorithm to use this dictionary for + */ +DENSITY_WINDOWS_EXPORT size_t density_get_dictionary_size(DENSITY_ALGORITHM algorithm); + +/* + * Return an output buffer byte size which guarantees enough space for encoding input_size bytes + * + * @param input_size the size of the input data which is about to be compressed + */ +DENSITY_WINDOWS_EXPORT uint_fast64_t density_compress_safe_size(const uint_fast64_t input_size); + +/* + * Return an output buffer byte size which, if expected_decompressed_output_size is correct, will enable density to decompress properly + * + * @param expected_decompressed_output_size the expected (original) size of the decompressed data + */ +DENSITY_WINDOWS_EXPORT uint_fast64_t density_decompress_safe_size(const uint_fast64_t expected_decompressed_output_size); + +/* + * Releases a context from memory. + * + * @param context the context to free + * @param mem_free the memory freeing function. If set to NULL, free() is used + */ +DENSITY_WINDOWS_EXPORT void density_free_context(density_context *const context, void (*mem_free)(void *)); + +/* + * Allocate a context in memory using the provided function and optional dictionary + * + * @param algorithm the required algorithm + * @param custom_dictionary use an eventual custom dictionary ? If set to true the context's dictionary will have to be allocated + * @param mem_alloc the memory allocation function. If set to NULL, malloc() is used + */ +DENSITY_WINDOWS_EXPORT density_processing_result density_compress_prepare_context(const DENSITY_ALGORITHM algorithm, const bool custom_dictionary, void *(*mem_alloc)(size_t)); + +/* + * Compress an input_buffer of input_size bytes and store the result in output_buffer, using the provided context. + * Important note * this function could be unsafe memory-wise if not used properly. + * + * @param input_buffer a buffer of bytes + * @param input_size the size in bytes of input_buffer + * @param output_buffer a buffer of bytes + * @param output_size the size of output_buffer, must be at least DENSITY_MINIMUM_OUTPUT_BUFFER_SIZE + * @param context a pointer to a context structure + */ +DENSITY_WINDOWS_EXPORT density_processing_result density_compress_with_context(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size, density_context *const context); + +/* + * Compress an input_buffer of input_size bytes and store the result in output_buffer. + * + * @param input_buffer a buffer of bytes + * @param input_size the size in bytes of input_buffer + * @param output_buffer a buffer of bytes + * @param output_size the size of output_buffer, must be at least DENSITY_MINIMUM_OUTPUT_BUFFER_SIZE + * @param algorithm the algorithm to use + */ +DENSITY_WINDOWS_EXPORT density_processing_result density_compress(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size, const DENSITY_ALGORITHM algorithm); + +/* + * Reads the compressed data's header and creates an adequate decompression context. + * + * @param input_buffer a buffer of bytes + * @param input_size the size in bytes of input_buffer + * @param custom_dictionary use a custom dictionary ? If set to true the context's dictionary will have to be allocated + * @param mem_alloc the memory allocation function. If set to NULL, malloc() is used + */ +DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_prepare_context(const uint8_t *input_buffer, const uint_fast64_t input_size, const bool custom_dictionary, void *(*mem_alloc)(size_t)); + +/* + * Decompress an input_buffer of input_size bytes and store the result in output_buffer, using the provided dictionary. + * Important notes * You must know in advance the algorithm used for compression to provide the proper dictionary. + * * This function could be unsafe memory-wise if not used properly. + * + * @param input_buffer a buffer of bytes + * @param input_size the size in bytes of input_buffer + * @param output_buffer a buffer of bytes + * @param output_size the size of output_buffer, must be at least DENSITY_MINIMUM_OUTPUT_BUFFER_SIZE + * @param dictionaries a pointer to a dictionary + */ +DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_with_context(const uint8_t * input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size, density_context *const context); + +/* + * Decompress an input_buffer of input_size bytes and store the result in output_buffer. + * + * @param input_buffer a buffer of bytes + * @param input_size the size in bytes of input_buffer + * @param output_buffer a buffer of bytes + * @param output_size the size of output_buffer, must be at least DENSITY_MINIMUM_OUTPUT_BUFFER_SIZE + */ +DENSITY_WINDOWS_EXPORT density_processing_result density_decompress(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/density/src/globals.c b/contrib/density/src/globals.c new file mode 100644 index 00000000000..4c58cd6f067 --- /dev/null +++ b/contrib/density/src/globals.c @@ -0,0 +1,47 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 01/11/13 13:39 + */ + +#include "globals.h" + +DENSITY_WINDOWS_EXPORT uint8_t density_version_major() { + return DENSITY_MAJOR_VERSION; +} + +DENSITY_WINDOWS_EXPORT uint8_t density_version_minor() { + return DENSITY_MINOR_VERSION; +} + +DENSITY_WINDOWS_EXPORT uint8_t density_version_revision() { + return DENSITY_REVISION; +} diff --git a/contrib/density/src/globals.h b/contrib/density/src/globals.h new file mode 100644 index 00000000000..eb44a52a673 --- /dev/null +++ b/contrib/density/src/globals.h @@ -0,0 +1,232 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 11/10/13 02:01 + */ + +#ifndef DENSITY_GLOBALS_H +#define DENSITY_GLOBALS_H + +#include +#include +#include +#include + +#include "density_api.h" + +#if defined(__clang__) || defined(__GNUC__) +#define DENSITY_FORCE_INLINE inline __attribute__((always_inline)) +#define DENSITY_RESTRICT restrict +#define DENSITY_RESTRICT_DECLARE +#define DENSITY_MEMCPY __builtin_memcpy +#define DENSITY_MEMMOVE __builtin_memmove +#define DENSITY_MEMSET __builtin_memset +#define DENSITY_LIKELY(x) __builtin_expect(!!(x), 1) +#define DENSITY_UNLIKELY(x) __builtin_expect(!!(x), 0) +#define DENSITY_PREFETCH(x) __builtin_prefetch(x) +#define DENSITY_CTZ(x) __builtin_ctz(x) + +#if defined(__BYTE_ORDER__) +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define DENSITY_LITTLE_ENDIAN +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define DENSITY_BIG_ENDIAN +#else +#error Unsupported endianness +#endif +#else +#error Unkwnown endianness +#endif + +#elif defined(_MSC_VER) +#include +#include + +#define DENSITY_FORCE_INLINE __forceinline +#define DENSITY_RESTRICT __restrict +#define DENSITY_RESTRICT_DECLARE __restrict +#define DENSITY_MEMCPY memcpy +#define DENSITY_MEMMOVE memmove +#define DENSITY_MEMSET memset +#define DENSITY_LIKELY(x) (x) +#define DENSITY_UNLIKELY(x) (x) +#define DENSITY_PREFETCH(x) ((void)(x)) + +DENSITY_FORCE_INLINE uint_fast8_t density_msvc_ctz(uint64_t value) { + unsigned long trailing_zero = 0; + if (_BitScanForward(&trailing_zero, (unsigned long)value)) + return (uint_fast8_t)trailing_zero; + else + return 0; +} +#define DENSITY_CTZ(x) density_msvc_ctz(x) + +#define DENSITY_LITTLE_ENDIAN // Little endian by default on Windows + +#else +#error Unsupported compiler +#endif + +#ifdef DENSITY_LITTLE_ENDIAN +#define DENSITY_LITTLE_ENDIAN_64(b) ((uint64_t)b) +#define DENSITY_LITTLE_ENDIAN_32(b) ((uint32_t)b) +#define DENSITY_LITTLE_ENDIAN_16(b) ((uint16_t)b) +#elif defined(DENSITY_BIG_ENDIAN) +#if __GNUC__ * 100 + __GNUC_MINOR__ >= 403 +#define DENSITY_LITTLE_ENDIAN_64(b) __builtin_bswap64(b) +#define DENSITY_LITTLE_ENDIAN_32(b) __builtin_bswap32(b) +#define DENSITY_LITTLE_ENDIAN_16(b) __builtin_bswap16(b) +#else +#warning Using bulk byte swap routines. Expect performance issues. +#define DENSITY_LITTLE_ENDIAN_64(b) ((((b) & 0xFF00000000000000ull) >> 56) | (((b) & 0x00FF000000000000ull) >> 40) | (((b) & 0x0000FF0000000000ull) >> 24) | (((b) & 0x000000FF00000000ull) >> 8) | (((b) & 0x00000000FF000000ull) << 8) | (((b) & 0x0000000000FF0000ull) << 24ull) | (((b) & 0x000000000000FF00ull) << 40) | (((b) & 0x00000000000000FFull) << 56)) +#define DENSITY_LITTLE_ENDIAN_32(b) ((((b) & 0xFF000000) >> 24) | (((b) & 0x00FF0000) >> 8) | (((b) & 0x0000FF00) << 8) | (((b) & 0x000000FF) << 24)) +#define DENSITY_LITTLE_ENDIAN_16(b) ((((b) & 0xFF00) >> 8) | (((b) & 0x00FF) << 8)) +#endif +#else +#error Unsupported endianness +#endif + +#define DENSITY_MAX_2(a, b) (((a)>(b))?(a):(b)) +#define DENSITY_MAX_3(a, b, c) (DENSITY_MAX_2(DENSITY_MAX_2(a, b), c)) + +#define DENSITY_FORMAT(v) 0##v##llu + +#define DENSITY_ISOLATE(b, p) ((DENSITY_FORMAT(b) / p) & 0x1) + +#define DENSITY_BINARY_TO_UINT(b) ((DENSITY_ISOLATE(b, 1llu) ? 0x1 : 0)\ + + (DENSITY_ISOLATE(b, 8llu) ? 0x2 : 0)\ + + (DENSITY_ISOLATE(b, 64llu) ? 0x4 : 0)\ + + (DENSITY_ISOLATE(b, 512llu) ? 0x8 : 0)\ + + (DENSITY_ISOLATE(b, 4096llu) ? 0x10 : 0)\ + + (DENSITY_ISOLATE(b, 32768llu) ? 0x20 : 0)\ + + (DENSITY_ISOLATE(b, 262144llu) ? 0x40 : 0)\ + + (DENSITY_ISOLATE(b, 2097152llu) ? 0x80 : 0)\ + + (DENSITY_ISOLATE(b, 16777216llu) ? 0x100 : 0)\ + + (DENSITY_ISOLATE(b, 134217728llu) ? 0x200 : 0)\ + + (DENSITY_ISOLATE(b, 1073741824llu) ? 0x400 : 0)\ + + (DENSITY_ISOLATE(b, 8589934592llu) ? 0x800 : 0)\ + + (DENSITY_ISOLATE(b, 68719476736llu) ? 0x1000 : 0)\ + + (DENSITY_ISOLATE(b, 549755813888llu) ? 0x2000 : 0)\ + + (DENSITY_ISOLATE(b, 4398046511104llu) ? 0x4000 : 0)\ + + (DENSITY_ISOLATE(b, 35184372088832llu) ? 0x8000 : 0)\ + + (DENSITY_ISOLATE(b, 281474976710656llu) ? 0x10000 : 0)\ + + (DENSITY_ISOLATE(b, 2251799813685248llu) ? 0x20000 : 0)) + +#define DENSITY_UNROLL_2(op) op; op +#define DENSITY_UNROLL_4(op) DENSITY_UNROLL_2(op); DENSITY_UNROLL_2(op) +#define DENSITY_UNROLL_8(op) DENSITY_UNROLL_4(op); DENSITY_UNROLL_4(op) +#define DENSITY_UNROLL_16(op) DENSITY_UNROLL_8(op); DENSITY_UNROLL_8(op) +#define DENSITY_UNROLL_32(op) DENSITY_UNROLL_16(op); DENSITY_UNROLL_16(op) +#define DENSITY_UNROLL_64(op) DENSITY_UNROLL_32(op); DENSITY_UNROLL_32(op) + +#define DENSITY_CASE_GENERATOR_2(op_a, flag_a, op_b, flag_b, op_mid, shift)\ + case ((flag_b << shift) | flag_a):\ + op_a;\ + op_mid;\ + op_b;\ + break; + +#define DENSITY_CASE_GENERATOR_4(op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ + case ((flag_d << (shift * 3)) | (flag_c << (shift * 2)) | (flag_b << shift) | flag_a):\ + op_a;\ + op_mid;\ + op_b;\ + op_mid;\ + op_c;\ + op_mid;\ + op_d;\ + break; + +#define DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_1, flag_1, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ + DENSITY_CASE_GENERATOR_2(op_1, flag_1, op_a, flag_a, op_mid, shift);\ + DENSITY_CASE_GENERATOR_2(op_1, flag_1, op_b, flag_b, op_mid, shift);\ + DENSITY_CASE_GENERATOR_2(op_1, flag_1, op_c, flag_c, op_mid, shift);\ + DENSITY_CASE_GENERATOR_2(op_1, flag_1, op_d, flag_d, op_mid, shift); + +#define DENSITY_CASE_GENERATOR_4_2_COMBINED(op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ + DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_a, flag_a, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_b, flag_b, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_c, flag_c, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_d, flag_d, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift); + +#define DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ + DENSITY_CASE_GENERATOR_4(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_a, flag_a, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_b, flag_b, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_c, flag_c, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_d, flag_d, op_mid, shift); + +#define DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_2, flag_2, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ + DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_a, flag_a, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_b, flag_b, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_c, flag_c, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_d, flag_d, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift); + +#define DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_1, flag_1, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ + DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_a, flag_a, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_b, flag_b, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_c, flag_c, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_d, flag_d, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift); + +#define DENSITY_CASE_GENERATOR_4_4_COMBINED(op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ + DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_a, flag_a, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_b, flag_b, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_c, flag_c, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ + DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_d, flag_d, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift); + +#define DENSITY_DICTIONARY_PREFERRED_RESET_CYCLE_SHIFT 6 +#define DENSITY_DICTIONARY_PREFERRED_RESET_CYCLE (1 << DENSITY_DICTIONARY_PREFERRED_RESET_CYCLE_SHIFT) + + +#define density_bitsizeof(x) (8 * sizeof(x)) + +#define DENSITY_SPOOKYHASH_SEED_1 (0xabc) +#define DENSITY_SPOOKYHASH_SEED_2 (0xdef) + +DENSITY_WINDOWS_EXPORT uint8_t density_version_major(); + +DENSITY_WINDOWS_EXPORT uint8_t density_version_minor(); + +DENSITY_WINDOWS_EXPORT uint8_t density_version_revision(); + + +/********************************************************************************************************************** + * * + * Global compile-time switches * + * * + **********************************************************************************************************************/ + +#define DENSITY_MAJOR_VERSION 0 +#define DENSITY_MINOR_VERSION 14 +#define DENSITY_REVISION 2 + + + +#endif diff --git a/contrib/density/src/structure/header.c b/contrib/density/src/structure/header.c new file mode 100644 index 00000000000..4306a5962cf --- /dev/null +++ b/contrib/density/src/structure/header.c @@ -0,0 +1,57 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 11/10/13 17:56 + */ + +#include "header.h" + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_header_read(const uint8_t **DENSITY_RESTRICT in, density_header *DENSITY_RESTRICT header) { + header->version[0] = *(*in); + header->version[1] = *(*in + 1); + header->version[2] = *(*in + 2); + header->algorithm = *(*in + 3); + + *in += sizeof(density_header); +} + +DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_header_write(uint8_t **DENSITY_RESTRICT out, const DENSITY_ALGORITHM algorithm) { + *(*out) = DENSITY_MAJOR_VERSION; + *(*out + 1) = DENSITY_MINOR_VERSION; + *(*out + 2) = DENSITY_REVISION; + *(*out + 3) = algorithm; + *(*out + 4) = 0; + *(*out + 5) = 0; + *(*out + 6) = 0; + *(*out + 7) = 0; + + *out += sizeof(density_header); +} diff --git a/contrib/density/src/structure/header.h b/contrib/density/src/structure/header.h new file mode 100644 index 00000000000..d4065b5600c --- /dev/null +++ b/contrib/density/src/structure/header.h @@ -0,0 +1,58 @@ +/* + * Density + * + * Copyright (c) 2013, Guillaume Voirin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * 11/10/13 02:06 + */ + +#ifndef DENSITY_FILE_HEADER_H +#define DENSITY_FILE_HEADER_H + +#include +#include + +#include "../globals.h" +#include "../density_api.h" + +#pragma pack(push) +#pragma pack(4) + +typedef struct { + density_byte version[3]; + density_byte algorithm; + density_byte reserved[4]; +} density_header; + +#pragma pack(pop) + +DENSITY_WINDOWS_EXPORT void density_header_read(const uint8_t ** DENSITY_RESTRICT_DECLARE, density_header * DENSITY_RESTRICT_DECLARE); +DENSITY_WINDOWS_EXPORT void density_header_write(uint8_t ** DENSITY_RESTRICT_DECLARE, const DENSITY_ALGORITHM); + +#endif diff --git a/contrib/lizard/.gitattributes b/contrib/lizard/.gitattributes new file mode 100644 index 00000000000..6212bd405b4 --- /dev/null +++ b/contrib/lizard/.gitattributes @@ -0,0 +1,21 @@ +# Set the default behavior +* text eol=lf + +# Explicitly declare source files +*.c text eol=lf +*.h text eol=lf + +# Denote files that should not be modified. +*.odt binary +*.png binary + +# Visual Studio +*.sln text eol=crlf +*.vcxproj* text eol=crlf +*.vcproj* text eol=crlf +*.suo binary +*.rc text eol=crlf + +# Windows +*.bat text eol=crlf +*.cmd text eol=crlf diff --git a/contrib/lizard/.gitignore b/contrib/lizard/.gitignore new file mode 100644 index 00000000000..dc30b0e5504 --- /dev/null +++ b/contrib/lizard/.gitignore @@ -0,0 +1,37 @@ +# Object files +*.o +*.ko + +# Libraries +*.lib +*.a + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app + +# IDE / editors files +*.suo +*.user +.clang_complete + +# Directories +_lizardbench/ +_visual/ +_codelite/ +_backup/ +_codelite_lz4/ + +# Archives +*.zip +*.liz + +*.txt +*.bat \ No newline at end of file diff --git a/contrib/lizard/.travis.yml b/contrib/lizard/.travis.yml new file mode 100644 index 00000000000..29f75591a31 --- /dev/null +++ b/contrib/lizard/.travis.yml @@ -0,0 +1,268 @@ +language: c +matrix: + fast_finish: true + include: + - name: macOS + os: osx + env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lizard CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang + + + # Ubuntu 18.04 + - name: Ubuntu 18.04, gcc-9, 64-bit and 32-bit tests + dist: bionic + script: + - CC=gcc-9; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + - CC=gcc-9; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror + before_install: + - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + - sudo apt-get -q update + - sudo apt-get --no-install-suggests --no-install-recommends -yq install gcc-multilib gcc-9 gcc-9-multilib + + - name: Ubuntu 18.04, clang-9, 64-bit tests + dist: bionic + script: + - CC=clang-9; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - sourceline: 'deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main' + key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' + packages: + - clang-9 + + + + # Ubuntu 16.04 gcc + - name: Ubuntu 16.04, gcc-9, 64-bit and 32-bit tests + dist: xenial + script: + - CC=gcc-9; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + - CC=gcc-9; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-multilib + - gcc-9 + - gcc-9-multilib + + - name: Ubuntu 16.04, gcc-8, 64-bit and 32-bit tests + dist: xenial + script: + - CC=gcc-8; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + - CC=gcc-8; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-multilib + - gcc-8 + - gcc-8-multilib + + - name: Ubuntu 16.04, gcc-7, 64-bit and 32-bit tests + dist: xenial + script: + - CC=gcc-7; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + - CC=gcc-7; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-multilib + - gcc-7 + - gcc-7-multilib + + - name: Ubuntu 16.04, gcc-6, 64-bit and 32-bit tests + dist: xenial + script: + - CC=gcc-6; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + - CC=gcc-6; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-multilib + - gcc-6 + - gcc-6-multilib + + - name: Ubuntu 16.04, gcc-5, 64-bit and 32-bit tests + dist: xenial + script: + - CC=gcc-5; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + - CC=gcc-5; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-multilib + - gcc-5 + - gcc-5-multilib + + - name: Ubuntu 16.04, gcc-4.8, 64-bit and 32-bit tests + dist: xenial + script: + - CC=gcc-4.8; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + - CC=gcc-4.8; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror + addons: + apt: + packages: + - gcc-multilib + - gcc-4.8 + - gcc-4.8-multilib + + + + # Ubuntu 16.04 clang + - name: Ubuntu 16.04, clang-9, 64-bit tests + dist: xenial + script: + - CC=clang-9; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - sourceline: 'deb https://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main' + key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' + packages: + - clang-9 + + - name: Ubuntu 16.04, clang-8, 64-bit tests + dist: xenial + script: + - CC=clang-8; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-xenial-8 + packages: + - clang-8 + + - name: Ubuntu 16.04, clang-7, 64-bit tests + dist: xenial + script: + - CC=clang-7; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-xenial-7 + packages: + - clang-7 + + - name: Ubuntu 16.04, clang-6.0, 64-bit tests + dist: xenial + script: + - CC=clang-6.0; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-xenial-6.0 + packages: + - clang-6.0 + + - name: Ubuntu 16.04, clang-5.0, 64-bit tests + dist: xenial + script: + - CC=clang-5.0; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-xenial-5.0 + packages: + - clang-5.0 + + - name: Ubuntu 16.04, clang-4.0, 64-bit tests + dist: xenial + script: + - CC=clang-4.0; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-xenial-4.0 + packages: + - clang-4.0 + + - name: Ubuntu 14.04, clang-3.8, 64-bit tests + dist: trusty + script: + - CC=clang-3.8; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-trusty-3.8 + packages: + - clang-3.8 + + + + - name: Ubuntu 16.04, misc tests + env: Cmd="make c_standards && make gpptest-native && make clean && make examples && make clean && make cmake && make clean && make travis-install && make clean && make clangtest-native" COMPILER=cc + dist: xenial + + - name: Ubuntu 14.04, arm and aarch64 + env: Cmd='make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static && make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static' COMPILER=arm-linux-gnueabi-gcc + dist: trusty + sudo: required + addons: + apt: + packages: + - qemu-system-arm + - qemu-user-static + - gcc-arm-linux-gnueabi + - libc6-dev-armel-cross + - gcc-aarch64-linux-gnu + - libc6-dev-arm64-cross + + - name: Ubuntu 14.04, powerpc and ppc64 + env: Cmd='make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64' COMPILER=powerpc-linux-gnu-gcc + dist: trusty + sudo: required + addons: + apt: + packages: + - qemu-system-ppc + - qemu-user-static + - gcc-powerpc-linux-gnu + + - name: Ubuntu 14.04, valgrind + env: Cmd='make -C tests test-mem MOREFLAGS=-DLIZARD_RESET_MEM' COMPILER=cc + dist: trusty + sudo: required + addons: + apt: + packages: + - valgrind + + + #- env: Ubu=14.04 Cmd='make sanitize' COMPILER=clang + # dist: trusty + # sudo: required + # addons: + # apt: + # packages: + # - valgrind + + #- env: Ubu=14.04 Cmd='make staticAnalyze' COMPILER=clang + # dist: trusty + # sudo: required + # addons: + # apt: + # packages: + # - clang + + +script: + - echo Cmd=$Cmd + - $COMPILER -v + - sh -c "$Cmd" diff --git a/contrib/lizard/LICENSE b/contrib/lizard/LICENSE new file mode 100644 index 00000000000..df1edbca85e --- /dev/null +++ b/contrib/lizard/LICENSE @@ -0,0 +1,15 @@ +This repository uses 2 different licenses : +- all files in the `lib` directory use a BSD 2-Clause license +- all other files use a GPLv2 license, unless explicitly stated otherwise + +Relevant license is reminded at the top of each source file, +and with the presence of COPYING or LICENSE file. + +This model emphasizes the fact that +only files in the `lib` directory are designed to be included into 3rd party projects. + +Other files, such as those from `programs` or `examples` directory, +are not intended to be compiled outside of their context. +They can serve as source of inspiration, +but they should not be copy/pasted into 3rd party projects, +as this scenario is not supported. diff --git a/contrib/lizard/NEWS b/contrib/lizard/NEWS new file mode 100644 index 00000000000..05d424656e9 --- /dev/null +++ b/contrib/lizard/NEWS @@ -0,0 +1,41 @@ +Lizard v1.0 +- LZ5 v2.0 was renamed to Lizard v1.0 +- improved compression speed with many small files + +LZ5 v2.0 +- this version is optimized for decompression speed (instead of ratio for previous versions) +- LZ5 v2.0 contains 4 compression methods: + fastLZ4 : compression levels -10...-19 are designed to give better decompression speed than LZ4 i.e. over 2000 MB/s + LIZv1 : compression levels -20...-29 are designed to give better ratio than LZ4 keeping 75% decompression speed + fastLZ4 + Huffman : compression levels -30...-39 add Huffman coding to fastLZ4 + LIZv1 + Huffman : compression levels -40...-49 give the best ratio (comparable to zlib and low levels of zstd/brotli) at decompression speed of 1000 MB/s + +LZ5 v1.5 +- introduced compatibility with Visual C++ 2010 and newer +- attached Visual Studio 2010 project +- thoroughly tested with 21 Travis CI and 7 AppVeyor CI tests +- fixed bug with reusing a context in lizard_frame.c (LizardF_compressBegin and Lizard_compress_HC_continue) +- fixed rare bug in match finder (concerns levels 4 - 15) + +LZ5 v1.4.1 +- fixed bug with a backward match extension (level 11 and 12) + +LZ5 v1.4 +- improved: levels from 13 to 15 (maximum compression ratio) +- added a new parser: LizardHC_optimal_price_bt +- updated documentation: lizard_Block_format.md and lizard_Frame_format.md +- changed lizard.exe: the "-B" option with block size [1-7] = 64KB, 256KB, 1MB, 4MB, 16MB, 64MB, 256MB (default : 4 = 4MB) + +LZ5 v1.3.3 +- added: new levels from 11 to 18 (maximum compression ratio) +- added: a new parser: LizardHC_optimal_price +- fixed: buffer-overflow during decompression (thanks to m^2) + +LZ5 r132 +- improved compression ratio +- added: new parsers: LizardHC_fast, LizardHC_price_fast, LizardHC_lowest_price +- added: a special 1-byte codeword for the last occured offset +- added: support for 3-byte long matches (MINMATCH = 3) + +LZ5 r131 +The first release based on LZ4 r132 dev diff --git a/contrib/lizard/README.md b/contrib/lizard/README.md new file mode 100644 index 00000000000..50dd737fb8d --- /dev/null +++ b/contrib/lizard/README.md @@ -0,0 +1,95 @@ +Lizard - efficient compression with very fast decompression +-------------------------------------------------------- + +Lizard (formerly LZ5) is a lossless compression algorithm which contains 4 compression methods: +- fastLZ4 : compression levels -10...-19 are designed to give better decompression speed than [LZ4] i.e. over 2000 MB/s +- LIZv1 : compression levels -20...-29 are designed to give better ratio than [LZ4] keeping 75% decompression speed +- fastLZ4 + Huffman : compression levels -30...-39 add Huffman coding to fastLZ4 +- LIZv1 + Huffman : compression levels -40...-49 give the best ratio (comparable to [zlib] and low levels of [zstd]/[brotli]) at decompression speed of 1000 MB/s + +Lizard library is based on frequently used [LZ4] library by Yann Collet but the Lizard compression format is not compatible with LZ4. +Lizard library is provided as open-source software using BSD 2-Clause license. +The high compression/decompression speed is achieved without any SSE and AVX extensions. + + +|Branch |Status | +|------------|---------| +|lz5_v1.5 | [![Build Status][travis15Badge]][travisLink] [![Build status][Appveyor15Badge]][AppveyorLink] | +|lizard | [![Build Status][travis20Badge]][travisLink] [![Build status][Appveyor20Badge]][AppveyorLink] | + +[travis15Badge]: https://travis-ci.org/inikep/lizard.svg?branch=lz5_v1.5 "Continuous Integration test suite" +[travis20Badge]: https://travis-ci.org/inikep/lizard.svg?branch=lizard "Continuous Integration test suite" +[travisLink]: https://travis-ci.org/inikep/lizard +[Appveyor15Badge]: https://ci.appveyor.com/api/projects/status/cqw7emcuqge369p0/branch/lz5_v1.5?svg=true "Visual test suite" +[Appveyor20Badge]: https://ci.appveyor.com/api/projects/status/cqw7emcuqge369p0/branch/lizard?svg=true "Visual test suite" +[AppveyorLink]: https://ci.appveyor.com/project/inikep/lizard +[LZ4]: https://github.com/lz4/lz4 +[zlib]: https://github.com/madler/zlib +[zstd]: https://github.com/facebook/zstd +[brotli]: https://github.com/google/brotli + + +Benchmarks +------------------------- + +The following results are obtained with [lzbench](https://github.com/inikep/lzbench) and `-t16,16` +using 1 core of Intel Core i5-4300U, Windows 10 64-bit (MinGW-w64 compilation under gcc 6.2.0) +with [silesia.tar] which contains tarred files from [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia). + +| Compressor name | Compression| Decompress.| Compr. size | Ratio | +| --------------- | -----------| -----------| ----------- | ----- | +| memcpy | 7332 MB/s | 8719 MB/s | 211947520 |100.00 | +| lz4 1.7.3 | 440 MB/s | 2318 MB/s | 100880800 | 47.60 | +| lz4hc 1.7.3 -1 | 98 MB/s | 2121 MB/s | 87591763 | 41.33 | +| lz4hc 1.7.3 -4 | 55 MB/s | 2259 MB/s | 79807909 | 37.65 | +| lz4hc 1.7.3 -9 | 22 MB/s | 2315 MB/s | 77892285 | 36.75 | +| lz4hc 1.7.3 -12 | 17 MB/s | 2323 MB/s | 77849762 | 36.73 | +| lz4hc 1.7.3 -16 | 10 MB/s | 2323 MB/s | 77841782 | 36.73 | +| lizard 1.0 -10 | 346 MB/s | 2610 MB/s | 103402971 | 48.79 | +| lizard 1.0 -12 | 103 MB/s | 2458 MB/s | 86232422 | 40.69 | +| lizard 1.0 -15 | 50 MB/s | 2552 MB/s | 81187330 | 38.31 | +| lizard 1.0 -19 | 3.04 MB/s | 2497 MB/s | 77416400 | 36.53 | +| lizard 1.0 -21 | 157 MB/s | 1795 MB/s | 89239174 | 42.10 | +| lizard 1.0 -23 | 30 MB/s | 1778 MB/s | 81097176 | 38.26 | +| lizard 1.0 -26 | 6.63 MB/s | 1734 MB/s | 74503695 | 35.15 | +| lizard 1.0 -29 | 1.37 MB/s | 1634 MB/s | 68694227 | 32.41 | +| lizard 1.0 -30 | 246 MB/s | 909 MB/s | 85727429 | 40.45 | +| lizard 1.0 -32 | 94 MB/s | 1244 MB/s | 76929454 | 36.30 | +| lizard 1.0 -35 | 47 MB/s | 1435 MB/s | 73850400 | 34.84 | +| lizard 1.0 -39 | 2.94 MB/s | 1502 MB/s | 69807522 | 32.94 | +| lizard 1.0 -41 | 126 MB/s | 961 MB/s | 76100661 | 35.91 | +| lizard 1.0 -43 | 28 MB/s | 1101 MB/s | 70955653 | 33.48 | +| lizard 1.0 -46 | 6.25 MB/s | 1073 MB/s | 65413061 | 30.86 | +| lizard 1.0 -49 | 1.27 MB/s | 1064 MB/s | 60679215 | 28.63 | +| zlib 1.2.8 -1 | 66 MB/s | 244 MB/s | 77259029 | 36.45 | +| zlib 1.2.8 -6 | 20 MB/s | 263 MB/s | 68228431 | 32.19 | +| zlib 1.2.8 -9 | 8.37 MB/s | 266 MB/s | 67644548 | 31.92 | +| zstd 1.1.1 -1 | 235 MB/s | 645 MB/s | 73659468 | 34.75 | +| zstd 1.1.1 -2 | 181 MB/s | 600 MB/s | 70168955 | 33.11 | +| zstd 1.1.1 -5 | 88 MB/s | 565 MB/s | 65002208 | 30.67 | +| zstd 1.1.1 -8 | 31 MB/s | 619 MB/s | 61026497 | 28.79 | +| zstd 1.1.1 -11 | 16 MB/s | 613 MB/s | 59523167 | 28.08 | +| zstd 1.1.1 -15 | 4.97 MB/s | 639 MB/s | 58007773 | 27.37 | +| zstd 1.1.1 -18 | 2.87 MB/s | 583 MB/s | 55294241 | 26.09 | +| zstd 1.1.1 -22 | 1.44 MB/s | 505 MB/s | 52731930 | 24.88 | +| brotli 0.5.2 -0 | 217 MB/s | 244 MB/s | 78226979 | 36.91 | +| brotli 0.5.2 -2 | 96 MB/s | 283 MB/s | 68066621 | 32.11 | +| brotli 0.5.2 -5 | 24 MB/s | 312 MB/s | 60801716 | 28.69 | +| brotli 0.5.2 -8 | 5.56 MB/s | 324 MB/s | 57382470 | 27.07 | +| brotli 0.5.2 -11 | 0.39 MB/s | 266 MB/s | 51138054 | 24.13 | + +[silesia.tar]: https://drive.google.com/file/d/0BwX7dtyRLxThenZpYU9zLTZhR1k/view?usp=sharing + + +Documentation +------------------------- + +The raw Lizard block compression format is detailed within [lizard_Block_format]. + +To compress an arbitrarily long file or data stream, multiple blocks are required. +Organizing these blocks and providing a common header format to handle their content +is the purpose of the Frame format, defined into [lizard_Frame_format]. +Interoperable versions of Lizard must respect this frame format. + +[lizard_Block_format]: doc/lizard_Block_format.md +[lizard_Frame_format]: doc/lizard_Frame_format.md diff --git a/contrib/lizard/appveyor.yml b/contrib/lizard/appveyor.yml new file mode 100644 index 00000000000..d5863f7c06b --- /dev/null +++ b/contrib/lizard/appveyor.yml @@ -0,0 +1,143 @@ +version: 1.0.{build} +environment: + matrix: + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "x64" + - COMPILER: "visual" + CONFIGURATION: "Debug" + PLATFORM: "Win32" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "x64" + - COMPILER: "visual" + CONFIGURATION: "Release" + PLATFORM: "Win32" + - COMPILER: "gcc" + PLATFORM: "mingw64" + - COMPILER: "gcc" + PLATFORM: "mingw32" + - COMPILER: "gcc" + PLATFORM: "clang" + +install: + - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% + - MKDIR bin + - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH% + - if [%COMPILER%]==[gcc] ( + SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && + SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && + COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe && + COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe + ) else ( + IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") + ) + +build_script: + - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% + - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% + - if [%PLATFORM%]==[clang] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% + - ECHO *** && + ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% && + ECHO *** + - if [%PLATFORM%]==[clang] (clang -v) + - if [%COMPILER%]==[gcc] (gcc -v) + - if [%COMPILER%]==[gcc] ( + echo ----- && + make -v && + echo ----- && + if not [%PLATFORM%]==[clang] ( + make -C programs lizard && make -C tests fullbench && make -C lib lib + ) ELSE ( + make -C programs lizard CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" && + make -C tests fullbench CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" && + make -C lib lib CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" + ) + ) + - if [%COMPILER%]==[gccX] if not [%PLATFORM%]==[clang] ( + MKDIR bin\dll bin\static bin\example bin\include && + COPY tests\fullbench.c bin\example\ && + COPY lib\xxhash.c bin\example\ && + COPY lib\xxhash.h bin\example\ && + COPY lib\lizard.h bin\include\ && + COPY lib\lizardhc.h bin\include\ && + COPY lib\lizard_frame.h bin\include\ && + COPY lib\liblizard.a bin\static\liblizard_static.lib && + COPY lib\dll\liblizard.* bin\dll\ && + COPY lib\dll\example\Makefile bin\example\ && + COPY lib\dll\example\fullbench-dll.* bin\example\ && + COPY lib\dll\example\README.md bin\ && + COPY programs\lizard.exe bin\lizard.exe + ) + - if [%COMPILER%]==[gccX] if [%PLATFORM%]==[mingw64] ( + 7z.exe a bin\lizard_x64.zip NEWS .\bin\lizard.exe .\bin\README.md .\bin\example .\bin\dll .\bin\static .\bin\include && + appveyor PushArtifact bin\lizard_x64.zip + ) + - if [%COMPILER%]==[gccX] if [%PLATFORM%]==[mingw32] ( + 7z.exe a bin\lizard_x86.zip NEWS .\bin\lizard.exe .\bin\README.md .\bin\example .\bin\dll .\bin\static .\bin\include && + appveyor PushArtifact bin\lizard_x86.zip + ) + - if [%COMPILER%]==[gcc] (COPY tests\fullbench.exe programs\) + - if [%COMPILER%]==[visual] ( + ECHO *** && + ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "visual\VS2010\lizard.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + ECHO *** && + ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "visual\VS2010\lizard.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + ECHO *** && + ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "visual\VS2010\lizard.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + ECHO *** && + ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "visual\VS2010\lizard.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + COPY visual\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe programs\ + ) + +test_script: + - ECHO *** && + ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% && + ECHO *** + - if not [%COMPILER%]==[unknown] ( + CD programs && + lizard -h && + lizard -i1b lizard.exe && + lizard -i1b5 lizard.exe && + lizard -i1b10 lizard.exe && + lizard -i1b15 lizard.exe && + echo ------- lizard tested ------- && + fullbench.exe -i1 fullbench.exe + ) + +artifacts: + - path: bin\lizard_x64.zip + - path: bin\lizard_x86.zip + +deploy: +- provider: GitHub + artifact: bin\lizard_x64.zip + auth_token: + secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4 + force_update: true + prerelease: true + on: + COMPILER: gcc + PLATFORM: "mingw64" + appveyor_repo_tag: true + branch: autobuild + +- provider: GitHub + artifact: bin\lizard_x86.zip + auth_token: + secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4 + force_update: true + prerelease: true + on: + COMPILER: gcc + PLATFORM: "mingw32" + appveyor_repo_tag: true + branch: autobuild diff --git a/contrib/lizard/contrib/djgpp/LICENSE b/contrib/lizard/contrib/djgpp/LICENSE new file mode 100644 index 00000000000..fee0d3b3e23 --- /dev/null +++ b/contrib/lizard/contrib/djgpp/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2014, lpsantil +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/contrib/lizard/contrib/djgpp/README.MD b/contrib/lizard/contrib/djgpp/README.MD new file mode 100644 index 00000000000..8dfc3a838c4 --- /dev/null +++ b/contrib/lizard/contrib/djgpp/README.MD @@ -0,0 +1,21 @@ +# lizard for DOS/djgpp +This file details on how to compile lizard.exe, and liblizard.a for use on DOS/djgpp using +Andrew Wu's build-djgpp cross compilers ([GH][0], [Binaries][1]) on OSX, Linux. + +## Setup +* Download a djgpp tarball [binaries][1] for your platform. +* Extract and install it (`tar jxvf djgpp-linux64-gcc492.tar.bz2`). Note the path. We'll assume `/home/user/djgpp`. +* Add the `bin` folder to your `PATH`. In bash, do `export PATH=/home/user/djgpp/bin:$PATH`. +* The `Makefile` in `contrib/djgpp/` sets up `CC`, `AR`, `LD` for you. So, `CC=i586-pc-msdosdjgpp-gcc`, `AR=i586-pc-msdosdjgpp-ar`, `LD=i586-pc-msdosdjgpp-gcc`. + +## Building Lizard for DOS +In the base dir of lizard and with `contrib/djgpp/Makefile`, try: +Try: +* `make -f contrib/djgpp/Makefile` +* `make -f contrib/djgpp/Makefile liblizard.a` +* `make -f contrib/djgpp/Makefile lizard.exe` +* `make -f contrib/djgpp/Makefile DESTDIR=/home/user/dos install`, however it doesn't make much sense on a \*nix. +* You can also do `make -f contrib/djgpp/Makefile uninstall` + +[0]: https://github.com/andrewwutw/build-djgpp +[1]: https://github.com/andrewwutw/build-djgpp/releases diff --git a/contrib/lizard/doc/lizard_Block_format.md b/contrib/lizard/doc/lizard_Block_format.md new file mode 100644 index 00000000000..45307a1fce5 --- /dev/null +++ b/contrib/lizard/doc/lizard_Block_format.md @@ -0,0 +1,181 @@ +Lizard v1.x Block Format Description +============================ +Last revised: 2016-10-08 +Authors : Yann Collet, Przemyslaw Skibinski + + +This specification is intended for developers +willing to produce Lizard-compatible compressed data blocks +using any programming language. + +Lizard is an LZ77-type compressor with a fixed, byte-oriented encoding. +There is no framing layer as it is assumed to be handled by other parts of the system (see [Lizard Frame format]). +This design is assumed to favor simplicity and speed. +It helps later on for optimizations, compactness, and features. + +This document describes only the block format, +not how the compressor nor decompressor actually work. +The correctness of the decompressor should not depend +on implementation details of the compressor, and vice versa. + +[Lizard Frame format]: lizard_Frame_format.md + + +Division into blocks +-------------------- + +The input data is divided into blocks of maximum size LIZARD_BLOCK_SIZE (which is 128 KB). The subsequent blocks use the same sliding window and are dependent on previous blocks. +Our impementation of Lizard compressor divides input data into blocks of size of LIZARD_BLOCK_SIZE except the last one which usually will be smaller. +The output data is a single byte 'Compression_Level' and one or more blocks in the format described below. + + +Block header format +----------------------- + +The block header is a single byte `Header_Byte` that is combination of following flags: + +| Name | Value | +| --------------------- | --- | +| LIZARD_FLAG_LITERALS | 1 | +| LIZARD_FLAG_FLAGS | 2 | +| LIZARD_FLAG_OFF16LEN | 4 | +| LIZARD_FLAG_OFF24LEN | 8 | +| LIZARD_FLAG_LEN | 16 | +| LIZARD_FLAG_UNCOMPRESSED | 128 | + +When `Header_Byte & LIZARD_FLAG_UNCOMPRESSED` is true then the block is followed by 3-byte `Uncompressed_length` and uncompressed data of given size. + + +Compressed block content +------------------------ + +When `Header_Byte & LIZARD_FLAG_UNCOMPRESSED` is false then compressed block contains of 5 streams: +- `Lengths_Stream` (compressed with Huffman if LIZARD_FLAG_LEN is set) +- `16-bit_Offsets_Stream` (compressed with Huffman if LIZARD_FLAG_OFF16LEN is set) +- `24-bit_Offsets_Stream` (compressed with Huffman if LIZARD_FLAG_OFF24LEN is set) +- `Tokens_Stream` (compressed with Huffman if LIZARD_FLAG_FLAGS is set) +- `Literals_Stream` (compressed with Huffman if LIZARD_FLAG_LITERALS is set) + + +Stream format +------------- +The single stream is either: +- if LIZARD_FLAG_XXX is not set: 3 byte `Stream_Length` followed by a given number bytes +- if LIZARD_FLAG_XXX is set: 3 byte `Original_Stream_Length`, 3 byte `Compressed_Stream_Length`, followed by a given number of Huffman compressed bytes + + +Lizard block decompression +----------------------- +At the beginning we have 5 streams and their sizes. +Decompressor should iterate through `Tokens_Stream`. Each token is 1-byte long and describes how to get data from other streams. +If token points a stream that is already empty it means that data is corrupted. + + +Lizard token decompression +----------------------- +The token is a one byte. Token decribes: +- how many literals should be copied from `Literals_Stream` +- if offset should be read from `16-bit_Offsets_Stream` or `24-bit_Offsets_Stream` +- how many bytes are part of a match and should be copied from a sliding window + +Lizard uses 4 types of tokens: +- [0_MMMM_LLL] - 3-bit literal length (0-7+), use offset from `16-bit_Offsets_Stream`, 4-bit match length (4-15+) +- [1_MMMM_LLL] - 3-bit literal length (0-7+), use last offset, 4-bit match length (0-15+) +- token 31 - no literal length, use offset from `24-bit_Offsets_Stream`, match length (47+) +- token 0-30 - no literal length, use offset from `24-bit_Offsets_Stream`, 31 match lengths (16-46) + +Lizard uses different output codewords and is not compatible with LZ4. LZ4 output codewords are 3 byte long (24-bit) and look as follows: +- LLLL_MMMM OOOOOOOO OOOOOOOO - 16-bit offset, 4-bit match length, 4-bit literal length + + +The format of `Lengths_Stream` +------------------------------ +`Lengths_Stream` contains lenghts in the the following format: +- when 'First_Byte' is < 254 then lenght is equal 'First_Byte' +- when 'First_Byte' is 254 then lenght is equal to value of 2-bytes after 'First_Byte' i.e. 0-65536 +- when 'First_Byte' is 255 then lenght is equal to value of 3-bytes after 'First_Byte' i.e. 0-16777215 + + +[0_MMMM_LLL] and [1_MMMM_LLL] tokens +--------------------------------------- +The length of literals to be copied from `Literals_Stream` depends on the literal length field (LLL) that uses 3 bits of the token. +Therefore each field ranges from 0 to 7. +If the value is 7, then the lenght is increased with a length taken from `Lengths_Stream`. + +Example 1 : A literal length of 48 will be represented as : + + - 7 : value for the 3-bits LLL field + - 41 : (=48-7) remaining length to reach 48 (in `Lengths_Stream`) + +Example 2 : A literal length of 280 for will be represented as : + + - 7 : value for the 3-bits LLL field + - 254 : informs that remaining length (=280-7) must be represented as 2-bytes (in `Lengths_Stream`) + - 273 : (=280-7) encoded as 2-bytes (in `Lengths_Stream`) + +Example 3 : A literal length of 7 for will be represented as : + + - 7 : value for the 3-bits LLL field + - 0 : (=7-7) yes, the zero must be output (in `Lengths_Stream`) + +After copying 0 or more literals from `Literals_Stream` we can prepare the match copy operation which depends on a offset and a match length. +The flag "0" informs that decoder should use the last encoded offset. +The flag "1" informs that the offset is a 2 bytes value (16-bit), in little endian format and should be taken from `16-bit_Offsets_Stream`. + +The match length depends on the match length field (MMMM) that uses 4 bits of the token. +Therefore each field ranges from 0 to 15. Values from 0-3 are forbidden with offset taken from `16-bit_Offsets_Stream`. +If the value is 15, then the lenght is increased with a length taken from `Lengths_Stream`. + +With the offset and the match length, +the decoder can now proceed to copy the data from the already decoded buffer. + + +Lizard block epilogue +------------------ +When all tokens are read from `Tokens_Stream` and interpreted all remaining streams should also be empty. +Otherwise, it means that the data is corrupted. The only exception is `Literals_Stream` that should have at least 16 remaining literals what +allows fast memory copy operations. The remaining literals up to the end of `Literals_Stream` should be appended to the output data. + + +Tokens 0-31 +----------- +The offset is a 3 bytes value (24-bit), in little endian format and should be taken from `24-bit_Offsets_Stream`. +The offset represents the position of the match to be copied from. +1 means "current position - 1 byte". +The maximum offset value is (1<<24)-1, 1<<24 cannot be coded. +Note that 0 is an invalid value, not used. + +The 'Token_Value' ranges from 0 to 31. +The match length is equal to 'Token_Value + 16 that is from 16 to 47. +If match length is 47, the lenght is increased with a length taken from `Lengths_Stream`. + + +Parsing restrictions +----------------------- +There are specific parsing rules to respect in order to remain compatible +with assumptions made by the decoder : + +1. The last 16 bytes are always literals what allows fast memory copy operations. +2. The last match must start at least 20 bytes before end of block. + Consequently, a block with less than 20 bytes cannot be compressed. + +These rules are in place to ensure that the decoder +will never read beyond the input buffer, nor write beyond the output buffer. + +Note that the last sequence is also incomplete, +and stops right after literals. + + +Additional notes +----------------------- +There is no assumption nor limits to the way the compressor +searches and selects matches within the source data block. +It could be a fast scan, a multi-probe, a full search using BST, +standard hash chains or MMC, well whatever. + +Advanced parsing strategies can also be implemented, such as lazy match, +or full optimal parsing. + +All these trade-off offer distinctive speed/memory/compression advantages. +Whatever the method used by the compressor, its result will be decodable +by any Lizard decoder if it follows the format specification described above. diff --git a/contrib/lizard/doc/lizard_Frame_format.md b/contrib/lizard/doc/lizard_Frame_format.md new file mode 100644 index 00000000000..ce24b0e23e2 --- /dev/null +++ b/contrib/lizard/doc/lizard_Frame_format.md @@ -0,0 +1,312 @@ +Lizard v1.x Frame Format Description +================================= + +###Notices + +Copyright (c) 2013-2015 Yann Collet + +Copyright (c) 2016 Przemyslaw Skibinski + +Permission is granted to copy and distribute this document +for any purpose and without charge, +including translations into other languages +and incorporation into compilations, +provided that the copyright notice and this notice are preserved, +and that any substantive changes or deletions from the original +are clearly marked. +Distribution of this document is unlimited. + +###Version + +1.0 (8-10-2016) + + +Introduction +------------ + +The purpose of this document is to define a lossless compressed data format, +that is independent of CPU type, operating system, +file system and character set, suitable for +File compression, Pipe and streaming compression +using the Lizard algorithm. + +The data can be produced or consumed, +even for an arbitrarily long sequentially presented input data stream, +using only an a priori bounded amount of intermediate storage, +and hence can be used in data communications. +The format uses the Lizard compression method, +and optional [xxHash-32 checksum method](https://github.com/Cyan4973/xxHash), +for detection of data corruption. + +The data format defined by this specification +does not attempt to allow random access to compressed data. + +This specification is intended for use by implementers of software +to compress data into Lizard format and/or decompress data from Lizard format. +The text of the specification assumes a basic background in programming +at the level of bits and other primitive data representations. + +Unless otherwise indicated below, +a compliant compressor must produce data sets +that conform to the specifications presented here. +It doesn’t need to support all options though. + +A compliant decompressor must be able to decompress +at least one working set of parameters +that conforms to the specifications presented here. +It may also ignore checksums. +Whenever it does not support a specific parameter within the compressed stream, +it must produce a non-ambiguous error code +and associated error message explaining which parameter is unsupported. + + +General Structure of Lizard Frame format +------------------------------------- + +| MagicNb | F. Descriptor | Block | (...) | EndMark | C. Checksum | +|:-------:|:-------------:| ----- | ----- | ------- | ----------- | +| 4 bytes | 3-11 bytes | | | 4 bytes | 0-4 bytes | + +__Magic Number__ + +4 Bytes, Little endian format. +Value : 0x184D2206 (it was 0x184D2204 for LZ4 and 0x184D2205 for LZ5 v1.x) + +__Frame Descriptor__ + +3 to 11 Bytes, to be detailed in the next part. +Most important part of the spec. + +__Data Blocks__ + +To be detailed later on. +That’s where compressed data is stored. + +__EndMark__ + +The flow of blocks ends when the last data block has a size of “0”. +The size is expressed as a 32-bits value. + +__Content Checksum__ + +Content Checksum verify that the full content has been decoded correctly. +The content checksum is the result +of [xxh32() hash function](https://github.com/Cyan4973/xxHash) +digesting the original (decoded) data as input, and a seed of zero. +Content checksum is only present when its associated flag +is set in the frame descriptor. +Content Checksum validates the result, +that all blocks were fully transmitted in the correct order and without error, +and also that the encoding/decoding process itself generated no distortion. +Its usage is recommended. + +__Frame Concatenation__ + +In some circumstances, it may be preferable to append multiple frames, +for example in order to add new data to an existing compressed file +without re-framing it. + +In such case, each frame has its own set of descriptor flags. +Each frame is considered independent. +The only relation between frames is their sequential order. + +The ability to decode multiple concatenated frames +within a single stream or file +is left outside of this specification. +As an example, the reference lizard command line utility behavior is +to decode all concatenated frames in their sequential order. + + +Frame Descriptor +---------------- + +| FLG | BD | (Content Size) | HC | +| ------- | ------- |:--------------:| ------- | +| 1 byte | 1 byte | 0 - 8 bytes | 1 byte | + +The descriptor uses a minimum of 3 bytes, +and up to 11 bytes depending on optional parameters. + +__FLG byte__ + +| BitNb | 7-6 | 5 | 4 | 3 | 2 | 1-0 | +| ------- | ------- | ------- | --------- | ------- | --------- | -------- | +|FieldName| Version | B.Indep | B.Checksum| C.Size | C.Checksum|*Reserved*| + + +__BD byte__ + +| BitNb | 7 | 6-5-4 | 3-2-1-0 | +| ------- | -------- | ------------ | -------- | +|FieldName|*Reserved*| Block MaxSize|*Reserved*| + +In the tables, bit 7 is highest bit, while bit 0 is lowest. + +__Version Number__ + +2-bits field, must be set to “01”. +Any other value cannot be decoded by this version of the specification. +Other version numbers will use different flag layouts. + +__Block Independence flag__ + +If this flag is set to “1”, blocks are independent. +If this flag is set to “0”, each block depends on previous ones +(up to Lizard window size, which is 16 MB). +In such case, it’s necessary to decode all blocks in sequence. + +Block dependency improves compression ratio, especially for small blocks. +On the other hand, it makes direct jumps or multi-threaded decoding impossible. + +__Block checksum flag__ + +If this flag is set, each data block will be followed by a 4-bytes checksum, +calculated by using the xxHash-32 algorithm on the raw (compressed) data block. +The intention is to detect data corruption (storage or transmission errors) +immediately, before decoding. +Block checksum usage is optional. + +__Content Size flag__ + +If this flag is set, the uncompressed size of data included within the frame +will be present as an 8 bytes unsigned little endian value, after the flags. +Content Size usage is optional. + +__Content checksum flag__ + +If this flag is set, a content checksum will be appended after the EndMark. + +Recommended value : “1” (content checksum is present) + +__Block Maximum Size__ + +This information is intended to help the decoder allocate memory. +Size here refers to the original (uncompressed) data size. +Block Maximum Size is one value among the following table : + +| 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | +| --- | ------ | ------ | ---- | ---- | ----- | ----- | ------ | +| N/A | 128 KB | 256 KB | 1 MB | 4 MB | 16 MB | 64 MB | 256 MB | + + +The decoder may refuse to allocate block sizes above a (system-specific) size. +Unused values may be used in a future revision of the spec. +A decoder conformant to the current version of the spec +is only able to decode blocksizes defined in this spec. + +__Reserved bits__ + +Value of reserved bits **must** be 0 (zero). +Reserved bit might be used in a future version of the specification, +typically enabling new optional features. +If this happens, a decoder respecting the current version of the specification +shall not be able to decode such a frame. + +__Content Size__ + +This is the original (uncompressed) size. +This information is optional, and only present if the associated flag is set. +Content size is provided using unsigned 8 Bytes, for a maximum of 16 HexaBytes. +Format is Little endian. +This value is informational, typically for display or memory allocation. +It can be skipped by a decoder, or used to validate content correctness. + +__Header Checksum__ + +One-byte checksum of combined descriptor fields, including optional ones. +The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF ` +using zero as a seed, +and the full Frame Descriptor as an input +(including optional fields when they are present). +A wrong checksum indicates an error in the descriptor. +Header checksum is informational and can be skipped. + + +Data Blocks +----------- + +| Block Size | data | (Block Checksum) | +|:----------:| ------ |:----------------:| +| 4 bytes | | 0 - 4 bytes | + + +__Block Size__ + +This field uses 4-bytes, format is little-endian. + +The highest bit is “1” if data in the block is uncompressed. + +The highest bit is “0” if data in the block is compressed by Lizard. + +All other bits give the size, in bytes, of the following data block +(the size does not include the block checksum if present). + +Block Size shall never be larger than Block Maximum Size. +Such a thing could happen for incompressible source data. +In such case, such a data block shall be passed in uncompressed format. + +__Data__ + +Where the actual data to decode stands. +It might be compressed or not, depending on previous field indications. +Uncompressed size of Data can be any size, up to “block maximum size”. +Note that data block is not necessarily full : +an arbitrary “flush” may happen anytime. Any block can be “partially filled”. + +__Block checksum__ + +Only present if the associated flag is set. +This is a 4-bytes checksum value, in little endian format, +calculated by using the xxHash-32 algorithm on the raw (undecoded) data block, +and a seed of zero. +The intention is to detect data corruption (storage or transmission errors) +before decoding. + +Block checksum is cumulative with Content checksum. + + +Skippable Frames +---------------- + +| Magic Number | Frame Size | User Data | +|:------------:|:----------:| --------- | +| 4 bytes | 4 bytes | | + +Skippable frames allow the integration of user-defined data +into a flow of concatenated frames. +Its design is pretty straightforward, +with the sole objective to allow the decoder to quickly skip +over user-defined data and continue decoding. + +For the purpose of facilitating identification, +it is discouraged to start a flow of concatenated frames with a skippable frame. +If there is a need to start such a flow with some user data +encapsulated into a skippable frame, +it’s recommended to start with a zero-byte Lizard frame +followed by a skippable frame. +This will make it easier for file type identifiers. + + +__Magic Number__ + +4 Bytes, Little endian format. +Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F. +All 16 values are valid to identify a skippable frame. + +__Frame Size__ + +This is the size, in bytes, of the following User Data +(without including the magic number nor the size field itself). +4 Bytes, Little endian format, unsigned 32-bits. +This means User Data can’t be bigger than (2^32-1) Bytes. + +__User Data__ + +User Data can be anything. Data will just be skipped by the decoder. + + + +Version changes +--------------- + +1.0 : based on LZ4 Frame Format Description 1.5.1 (31/03/2015) diff --git a/contrib/lizard/examples/.gitignore b/contrib/lizard/examples/.gitignore new file mode 100644 index 00000000000..f6f8bbcca09 --- /dev/null +++ b/contrib/lizard/examples/.gitignore @@ -0,0 +1,8 @@ +/Makefile.liz* +/printVersion +/doubleBuffer +/ringBuffer +/ringBufferHC +/lineCompress +/frameCompress +/*.exe diff --git a/contrib/lizard/examples/HCStreaming_ringBuffer.c b/contrib/lizard/examples/HCStreaming_ringBuffer.c new file mode 100644 index 00000000000..3e4d75a5d78 --- /dev/null +++ b/contrib/lizard/examples/HCStreaming_ringBuffer.c @@ -0,0 +1,241 @@ +// Lizard streaming API example : ring buffer +// Based on previous work from Takayuki Matsuoka + + +/************************************** + * Compiler Options + **************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS // for MSVC +# define snprintf sprintf_s +#endif + +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +#endif + + +/************************************** + * Includes + **************************************/ +#include "lizard_compress.h" +#include "lizard_decompress.h" +#include "lizard_common.h" + +#include +#include +#include +#include + +enum { + MESSAGE_MAX_BYTES = 1024, + RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES, + DEC_BUFFER_BYTES = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES // Intentionally larger to test unsynchronized ring buffers +}; + + +size_t write_int32(FILE* fp, int32_t i) { + return fwrite(&i, sizeof(i), 1, fp); +} + +size_t write_bin(FILE* fp, const void* array, int arrayBytes) { + return fwrite(array, 1, arrayBytes, fp); +} + +size_t read_int32(FILE* fp, int32_t* i) { + return fread(i, sizeof(*i), 1, fp); +} + +size_t read_bin(FILE* fp, void* array, int arrayBytes) { + return fread(array, 1, arrayBytes, fp); +} + + +void test_compress(FILE* outFp, FILE* inpFp) +{ + Lizard_stream_t* lizardStream = Lizard_createStream(0); + static char inpBuf[RING_BUFFER_BYTES]; + int inpOffset = 0; + + if (!lizardStream) return; + + for(;;) + { + // Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer. + char* const inpPtr = &inpBuf[inpOffset]; + const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1; + const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); + if (0 == inpBytes) break; + + { + char cmpBuf[LIZARD_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; + const int cmpBytes = Lizard_compress_continue(lizardStream, inpPtr, cmpBuf, inpBytes, Lizard_compressBound(inpBytes)); + + if(cmpBytes <= 0) break; + write_int32(outFp, cmpBytes); + write_bin(outFp, cmpBuf, cmpBytes); + + inpOffset += inpBytes; + + // Wraparound the ringbuffer offset + if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) + inpOffset = 0; + } + } + + write_int32(outFp, 0); + Lizard_freeStream(lizardStream); +} + + +void test_decompress(FILE* outFp, FILE* inpFp) +{ + static char decBuf[DEC_BUFFER_BYTES]; + int decOffset = 0; + Lizard_streamDecode_t lizardStreamDecode_body = { 0 }; + Lizard_streamDecode_t* lizardStreamDecode = &lizardStreamDecode_body; + + for(;;) + { + int cmpBytes = 0; + char cmpBuf[LIZARD_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; + + { + const size_t r0 = read_int32(inpFp, &cmpBytes); + size_t r1; + if(r0 != 1 || cmpBytes <= 0) + break; + + r1 = read_bin(inpFp, cmpBuf, cmpBytes); + if(r1 != (size_t) cmpBytes) + break; + } + + { + char* const decPtr = &decBuf[decOffset]; + const int decBytes = Lizard_decompress_safe_continue( + lizardStreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES); + if(decBytes <= 0) + break; + + decOffset += decBytes; + write_bin(outFp, decPtr, decBytes); + + // Wraparound the ringbuffer offset + if(decOffset >= DEC_BUFFER_BYTES - MESSAGE_MAX_BYTES) + decOffset = 0; + } + } +} + + +// Compare 2 files content +// return 0 if identical +// return ByteNb>0 if different +size_t compare(FILE* f0, FILE* f1) +{ + size_t result = 1; + + for (;;) + { + char b0[65536]; + char b1[65536]; + const size_t r0 = fread(b0, 1, sizeof(b0), f0); + const size_t r1 = fread(b1, 1, sizeof(b1), f1); + + if ((r0==0) && (r1==0)) return 0; // success + + if (r0 != r1) + { + size_t smallest = r0; + if (r1 +#include +#include +#include + +enum { + BLOCK_BYTES = 1024 * 8, +// BLOCK_BYTES = 1024 * 64, +}; + + +size_t write_int(FILE* fp, int i) { + return fwrite(&i, sizeof(i), 1, fp); +} + +size_t write_bin(FILE* fp, const void* array, size_t arrayBytes) { + return fwrite(array, 1, arrayBytes, fp); +} + +size_t read_int(FILE* fp, int* i) { + return fread(i, sizeof(*i), 1, fp); +} + +size_t read_bin(FILE* fp, void* array, size_t arrayBytes) { + return fread(array, 1, arrayBytes, fp); +} + + +void test_compress(FILE* outFp, FILE* inpFp) +{ + Lizard_stream_t* lizardStream = Lizard_createStream_MinLevel(); + char inpBuf[2][BLOCK_BYTES]; + int inpBufIndex = 0; + + if (!lizardStream) return; + lizardStream = Lizard_resetStream_MinLevel(lizardStream); + if (!lizardStream) return; + + for(;;) { + char* const inpPtr = inpBuf[inpBufIndex]; + const int inpBytes = (int) read_bin(inpFp, inpPtr, BLOCK_BYTES); + if(0 == inpBytes) { + break; + } + + { + char cmpBuf[LIZARD_COMPRESSBOUND(BLOCK_BYTES)]; + const int cmpBytes = Lizard_compress_continue(lizardStream, inpPtr, cmpBuf, inpBytes, sizeof(cmpBuf)); + if(cmpBytes <= 0) { + break; + } + write_int(outFp, cmpBytes); + write_bin(outFp, cmpBuf, (size_t) cmpBytes); + } + + inpBufIndex = (inpBufIndex + 1) % 2; + } + + write_int(outFp, 0); + Lizard_freeStream(lizardStream); +} + + +void test_decompress(FILE* outFp, FILE* inpFp) +{ + Lizard_streamDecode_t lizardStreamDecode_body; + Lizard_streamDecode_t* lizardStreamDecode = &lizardStreamDecode_body; + + char decBuf[2][BLOCK_BYTES]; + int decBufIndex = 0; + + Lizard_setStreamDecode(lizardStreamDecode, NULL, 0); + + for(;;) { + char cmpBuf[LIZARD_COMPRESSBOUND(BLOCK_BYTES)]; + int cmpBytes = 0; + + { + const size_t readCount0 = read_int(inpFp, &cmpBytes); + if(readCount0 != 1 || cmpBytes <= 0) { + break; + } + + const size_t readCount1 = read_bin(inpFp, cmpBuf, (size_t) cmpBytes); + if(readCount1 != (size_t) cmpBytes) { + break; + } + } + + { + char* const decPtr = decBuf[decBufIndex]; + const int decBytes = Lizard_decompress_safe_continue( + lizardStreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES); + if(decBytes <= 0) { + break; + } + write_bin(outFp, decPtr, (size_t) decBytes); + } + + decBufIndex = (decBufIndex + 1) % 2; + } +} + + +int compare(FILE* fp0, FILE* fp1) +{ + int result = 0; + + while(0 == result) { + char b0[65536]; + char b1[65536]; + const size_t r0 = read_bin(fp0, b0, sizeof(b0)); + const size_t r1 = read_bin(fp1, b1, sizeof(b1)); + + result = (int) r0 - (int) r1; + + if(0 == r0 || 0 == r1) { + break; + } + if(0 == result) { + result = memcmp(b0, b1, r0); + } + } + + return result; +} + + +int main(int argc, char* argv[]) +{ + char inpFilename[256] = { 0 }; + char lizardFilename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if(argc < 2) { + printf("Please specify input filename\n"); + return 0; + } + + snprintf(inpFilename, 256, "%s", argv[1]); + snprintf(lizardFilename, 256, "%s.lizs-%d", argv[1], BLOCK_BYTES); + snprintf(decFilename, 256, "%s.lizs-%d.dec", argv[1], BLOCK_BYTES); + + printf("inp = [%s]\n", inpFilename); + printf("lizard = [%s]\n", lizardFilename); + printf("dec = [%s]\n", decFilename); + + // compress + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* outFp = fopen(lizardFilename, "wb"); + + printf("compress : %s -> %s\n", inpFilename, lizardFilename); + test_compress(outFp, inpFp); + printf("compress : done\n"); + + fclose(outFp); + fclose(inpFp); + } + + // decompress + { + FILE* inpFp = fopen(lizardFilename, "rb"); + FILE* outFp = fopen(decFilename, "wb"); + + printf("decompress : %s -> %s\n", lizardFilename, decFilename); + test_decompress(outFp, inpFp); + printf("decompress : done\n"); + + fclose(outFp); + fclose(inpFp); + } + + // verify + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* decFp = fopen(decFilename, "rb"); + + printf("verify : %s <-> %s\n", inpFilename, decFilename); + const int cmp = compare(inpFp, decFp); + if(0 == cmp) { + printf("verify : OK\n"); + } else { + printf("verify : NG\n"); + } + + fclose(decFp); + fclose(inpFp); + } + + return 0; +} diff --git a/contrib/lizard/examples/blockStreaming_doubleBuffer.md b/contrib/lizard/examples/blockStreaming_doubleBuffer.md new file mode 100644 index 00000000000..c27afcfb98c --- /dev/null +++ b/contrib/lizard/examples/blockStreaming_doubleBuffer.md @@ -0,0 +1,100 @@ +# Lizard streaming API Example : Double Buffer +by *Takayuki Matsuoka* + +`blockStreaming_doubleBuffer.c` is Lizard Straming API example which implements double buffer (de)compression. + +Please note : + + - Firstly, read "Lizard Streaming API Basics". + - This is relatively advanced application example. + - Output file is not compatible with lizard_frame and platform dependent. + + +## What's the point of this example ? + + - Handle huge file in small amount of memory + - Always better compression ratio than Block API + - Uniform block size + + +## How the compression works + +First of all, allocate "Double Buffer" for input and Lizard compressed data buffer for output. +Double buffer has two pages, "first" page (Page#1) and "second" page (Page#2). + +``` + Double Buffer + + Page#1 Page#2 + +---------+---------+ + | Block#1 | | + +----+----+---------+ + | + v + {Out#1} + + + Prefix Dependency + +---------+ + | | + v | + +---------+----+----+ + | Block#1 | Block#2 | + +---------+----+----+ + | + v + {Out#2} + + + External Dictionary Mode + +---------+ + | | + | v + +----+----+---------+ + | Block#3 | Block#2 | + +----+----+---------+ + | + v + {Out#3} + + + Prefix Dependency + +---------+ + | | + v | + +---------+----+----+ + | Block#3 | Block#4 | + +---------+----+----+ + | + v + {Out#4} +``` + +Next, read first block to double buffer's first page. And compress it by `Lizard_compress_continue()`. +For the first time, Lizard doesn't know any previous dependencies, +so it just compress the line without dependencies and generates compressed block {Out#1} to Lizard compressed data buffer. +After that, write {Out#1} to the file. + +Next, read second block to double buffer's second page. And compress it. +In this time, Lizard can use dependency to Block#1 to improve compression ratio. +This dependency is called "Prefix mode". + +Next, read third block to double buffer's *first* page. And compress it. +Also this time, Lizard can use dependency to Block#2. +This dependency is called "External Dictonaly mode". + +Continue these procedure to the end of the file. + + +## How the decompression works + +Decompression will do reverse order. + + - Read first compressed block. + - Decompress it to the first page and write that page to the file. + - Read second compressed block. + - Decompress it to the second page and write that page to the file. + - Read third compressed block. + - Decompress it to the *first* page and write that page to the file. + +Continue these procedure to the end of the compressed file. diff --git a/contrib/lizard/examples/blockStreaming_lineByLine.c b/contrib/lizard/examples/blockStreaming_lineByLine.c new file mode 100644 index 00000000000..9d239533501 --- /dev/null +++ b/contrib/lizard/examples/blockStreaming_lineByLine.c @@ -0,0 +1,210 @@ +// Lizard streaming API example : line-by-line logfile compression +// Copyright : Takayuki Matsuoka + + +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS +# define snprintf sprintf_s +#endif +#include "lizard_common.h" +#include "lizard_decompress.h" +#include +#include +#include +#include + +static size_t write_uint16(FILE* fp, uint16_t i) +{ + return fwrite(&i, sizeof(i), 1, fp); +} + +static size_t write_bin(FILE* fp, const void* array, int arrayBytes) +{ + return fwrite(array, 1, arrayBytes, fp); +} + +static size_t read_uint16(FILE* fp, uint16_t* i) +{ + return fread(i, sizeof(*i), 1, fp); +} + +static size_t read_bin(FILE* fp, void* array, int arrayBytes) +{ + return fread(array, 1, arrayBytes, fp); +} + + +static void test_compress( + FILE* outFp, + FILE* inpFp, + size_t messageMaxBytes, + size_t ringBufferBytes) +{ + Lizard_stream_t* const lizardStream = Lizard_createStream_MinLevel(); + const size_t cmpBufBytes = LIZARD_COMPRESSBOUND(messageMaxBytes); + char* const cmpBuf = (char*) malloc(cmpBufBytes); + char* const inpBuf = (char*) malloc(ringBufferBytes); + int inpOffset = 0; + + for ( ; ; ) + { + char* const inpPtr = &inpBuf[inpOffset]; + +#if 0 + // Read random length data to the ring buffer. + const int randomLength = (rand() % messageMaxBytes) + 1; + const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); + if (0 == inpBytes) break; +#else + // Read line to the ring buffer. + int inpBytes = 0; + if (!fgets(inpPtr, (int) messageMaxBytes, inpFp)) + break; + inpBytes = (int) strlen(inpPtr); +#endif + + { + const int cmpBytes = Lizard_compress_continue(lizardStream, inpPtr, cmpBuf, inpBytes, cmpBufBytes); + if (cmpBytes <= 0) break; + write_uint16(outFp, (uint16_t) cmpBytes); + write_bin(outFp, cmpBuf, cmpBytes); + + // Add and wraparound the ringbuffer offset + inpOffset += inpBytes; + if ((size_t)inpOffset >= ringBufferBytes - messageMaxBytes) inpOffset = 0; + } + } + write_uint16(outFp, 0); + + free(inpBuf); + free(cmpBuf); + Lizard_freeStream(lizardStream); +} + + +static void test_decompress( + FILE* outFp, + FILE* inpFp, + size_t messageMaxBytes, + size_t ringBufferBytes) +{ + Lizard_streamDecode_t* const lizardStreamDecode = Lizard_createStreamDecode(); + char* const cmpBuf = (char*) malloc(LIZARD_COMPRESSBOUND(messageMaxBytes)); + char* const decBuf = (char*) malloc(ringBufferBytes); + int decOffset = 0; + + for ( ; ; ) + { + uint16_t cmpBytes = 0; + + if (read_uint16(inpFp, &cmpBytes) != 1) break; + if (cmpBytes <= 0) break; + if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break; + + { + char* const decPtr = &decBuf[decOffset]; + const int decBytes = Lizard_decompress_safe_continue( + lizardStreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes); + if (decBytes <= 0) break; + write_bin(outFp, decPtr, decBytes); + + // Add and wraparound the ringbuffer offset + decOffset += decBytes; + if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0; + } + } + + free(decBuf); + free(cmpBuf); + Lizard_freeStreamDecode(lizardStreamDecode); +} + + +static int compare(FILE* f0, FILE* f1) +{ + int result = 0; + const size_t tempBufferBytes = 65536; + char* const b0 = (char*) malloc(tempBufferBytes); + char* const b1 = (char*) malloc(tempBufferBytes); + + while(0 == result) + { + const size_t r0 = fread(b0, 1, tempBufferBytes, f0); + const size_t r1 = fread(b1, 1, tempBufferBytes, f1); + + result = (int) r0 - (int) r1; + + if (0 == r0 || 0 == r1) break; + if (0 == result) result = memcmp(b0, b1, r0); + } + + free(b1); + free(b0); + return result; +} + + +int main(int argc, char* argv[]) +{ + enum { + MESSAGE_MAX_BYTES = 1024, + RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES, + }; + + char inpFilename[256] = { 0 }; + char lizardFilename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if (argc < 2) + { + printf("Please specify input filename\n"); + return 0; + } + + snprintf(inpFilename, 256, "%s", argv[1]); + snprintf(lizardFilename, 256, "%s.lizs", argv[1]); + snprintf(decFilename, 256, "%s.lizs.dec", argv[1]); + + printf("inp = [%s]\n", inpFilename); + printf("lizard = [%s]\n", lizardFilename); + printf("dec = [%s]\n", decFilename); + + // compress + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* outFp = fopen(lizardFilename, "wb"); + + test_compress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES); + + fclose(outFp); + fclose(inpFp); + } + + // decompress + { + FILE* inpFp = fopen(lizardFilename, "rb"); + FILE* outFp = fopen(decFilename, "wb"); + + test_decompress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES); + + fclose(outFp); + fclose(inpFp); + } + + // verify + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* decFp = fopen(decFilename, "rb"); + + const int cmp = compare(inpFp, decFp); + if (0 == cmp) + printf("Verify : OK\n"); + else + printf("Verify : NG\n"); + + fclose(decFp); + fclose(inpFp); + } + + return 0; +} diff --git a/contrib/lizard/examples/blockStreaming_lineByLine.md b/contrib/lizard/examples/blockStreaming_lineByLine.md new file mode 100644 index 00000000000..ac56b879c90 --- /dev/null +++ b/contrib/lizard/examples/blockStreaming_lineByLine.md @@ -0,0 +1,122 @@ +# Lizard streaming API Example : Line by Line Text Compression +by *Takayuki Matsuoka* + +`blockStreaming_lineByLine.c` is Lizard Straming API example which implements line by line incremental (de)compression. + +Please note the following restrictions : + + - Firstly, read "Lizard Streaming API Basics". + - This is relatively advanced application example. + - Output file is not compatible with lizard_frame and platform dependent. + + +## What's the point of this example ? + + - Line by line incremental (de)compression. + - Handle huge file in small amount of memory + - Generally better compression ratio than Block API + - Non-uniform block size + + +## How the compression works + +First of all, allocate "Ring Buffer" for input and Lizard compressed data buffer for output. + +``` +(1) + Ring Buffer + + +--------+ + | Line#1 | + +---+----+ + | + v + {Out#1} + + +(2) + Prefix Mode Dependency + +----+ + | | + v | + +--------+-+------+ + | Line#1 | Line#2 | + +--------+---+----+ + | + v + {Out#2} + + +(3) + Prefix Prefix + +----+ +----+ + | | | | + v | v | + +--------+-+------+-+------+ + | Line#1 | Line#2 | Line#3 | + +--------+--------+---+----+ + | + v + {Out#3} + + +(4) + External Dictionary Mode + +----+ +----+ + | | | | + v | v | + ------+--------+-+------+-+--------+ + | .... | Line#X | Line#X+1 | + ------+--------+--------+-----+----+ + ^ | + | v + | {Out#X+1} + | + Reset + + +(5) + Prefix + +-----+ + | | + v | + ------+--------+--------+----------+--+-------+ + | .... | Line#X | Line#X+1 | Line#X+2 | + ------+--------+--------+----------+-----+----+ + ^ | + | v + | {Out#X+2} + | + Reset +``` + +Next (see (1)), read first line to ringbuffer and compress it by `Lizard_compress_continue()`. +For the first time, Lizard doesn't know any previous dependencies, +so it just compress the line without dependencies and generates compressed line {Out#1} to Lizard compressed data buffer. +After that, write {Out#1} to the file and forward ringbuffer offset. + +Do the same things to second line (see (2)). +But in this time, Lizard can use dependency to Line#1 to improve compression ratio. +This dependency is called "Prefix mode". + +Eventually, we'll reach end of ringbuffer at Line#X (see (4)). +This time, we should reset ringbuffer offset. +After resetting, at Line#X+1 pointer is not adjacent, but Lizard still maintain its memory. +This is called "External Dictionary Mode". + +In Line#X+2 (see (5)), finally Lizard forget almost all memories but still remains Line#X+1. +This is the same situation as Line#2. + +Continue these procedure to the end of text file. + + +## How the decompression works + +Decompression will do reverse order. + + - Read compressed line from the file to buffer. + - Decompress it to the ringbuffer. + - Output decompressed plain text line to the file. + - Forward ringbuffer offset. If offset exceedes end of the ringbuffer, reset it. + +Continue these procedure to the end of the compressed file. diff --git a/contrib/lizard/examples/blockStreaming_ringBuffer.c b/contrib/lizard/examples/blockStreaming_ringBuffer.c new file mode 100644 index 00000000000..011cd8b665b --- /dev/null +++ b/contrib/lizard/examples/blockStreaming_ringBuffer.c @@ -0,0 +1,202 @@ +// Lizard streaming API example : ring buffer +// Based on sample code from Takayuki Matsuoka + + +/************************************** + * Compiler Options + **************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS // for MSVC +# define snprintf sprintf_s +#endif +#ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +#endif + + +/************************************** + * Includes + **************************************/ +#include +#include +#include +#include +#include "lizard_common.h" +#include "lizard_decompress.h" + + +enum { + MESSAGE_MAX_BYTES = 1024, + RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES, + DECODE_RING_BUFFER = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES // Intentionally larger, to test unsynchronized ring buffers +}; + + +size_t write_int32(FILE* fp, int32_t i) { + return fwrite(&i, sizeof(i), 1, fp); +} + +size_t write_bin(FILE* fp, const void* array, int arrayBytes) { + return fwrite(array, 1, arrayBytes, fp); +} + +size_t read_int32(FILE* fp, int32_t* i) { + return fread(i, sizeof(*i), 1, fp); +} + +size_t read_bin(FILE* fp, void* array, int arrayBytes) { + return fread(array, 1, arrayBytes, fp); +} + + +void test_compress(FILE* outFp, FILE* inpFp) +{ + Lizard_stream_t* lizardStream = Lizard_createStream_MinLevel(); + if (!lizardStream) return; + + static char inpBuf[RING_BUFFER_BYTES]; + int inpOffset = 0; + + for(;;) { + // Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer. + char* const inpPtr = &inpBuf[inpOffset]; + const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1; + const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); + if (0 == inpBytes) break; + + { + char cmpBuf[LIZARD_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; + const int cmpBytes = Lizard_compress_continue(lizardStream, inpPtr, cmpBuf, inpBytes, Lizard_compressBound(inpBytes)); + if(cmpBytes <= 0) break; + write_int32(outFp, cmpBytes); + write_bin(outFp, cmpBuf, cmpBytes); + + inpOffset += inpBytes; + + // Wraparound the ringbuffer offset + if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) inpOffset = 0; + } + } + + write_int32(outFp, 0); + Lizard_freeStream(lizardStream); +} + + +void test_decompress(FILE* outFp, FILE* inpFp) +{ + static char decBuf[DECODE_RING_BUFFER]; + int decOffset = 0; + Lizard_streamDecode_t lizardStreamDecode_body = { 0 }; + Lizard_streamDecode_t* lizardStreamDecode = &lizardStreamDecode_body; + + for(;;) { + int cmpBytes = 0; + char cmpBuf[LIZARD_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; + + { + const size_t r0 = read_int32(inpFp, &cmpBytes); + if(r0 != 1 || cmpBytes <= 0) break; + + const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes); + if(r1 != (size_t) cmpBytes) break; + } + + { + char* const decPtr = &decBuf[decOffset]; + const int decBytes = Lizard_decompress_safe_continue( + lizardStreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES); + if(decBytes <= 0) break; + decOffset += decBytes; + write_bin(outFp, decPtr, decBytes); + + // Wraparound the ringbuffer offset + if(decOffset >= DECODE_RING_BUFFER - MESSAGE_MAX_BYTES) decOffset = 0; + } + } +} + + +int compare(FILE* f0, FILE* f1) +{ + int result = 0; + + while(0 == result) { + char b0[65536]; + char b1[65536]; + const size_t r0 = fread(b0, 1, sizeof(b0), f0); + const size_t r1 = fread(b1, 1, sizeof(b1), f1); + + result = (int) r0 - (int) r1; + + if(0 == r0 || 0 == r1) { + break; + } + if(0 == result) { + result = memcmp(b0, b1, r0); + } + } + + return result; +} + + +int main(int argc, char** argv) +{ + char inpFilename[256] = { 0 }; + char lizardFilename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if(argc < 2) { + printf("Please specify input filename\n"); + return 0; + } + + snprintf(inpFilename, 256, "%s", argv[1]); + snprintf(lizardFilename, 256, "%s.lizs-%d", argv[1], 0); + snprintf(decFilename, 256, "%s.lizs-%d.dec", argv[1], 0); + + printf("inp = [%s]\n", inpFilename); + printf("lizard = [%s]\n", lizardFilename); + printf("dec = [%s]\n", decFilename); + + // compress + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* outFp = fopen(lizardFilename, "wb"); + + test_compress(outFp, inpFp); + + fclose(outFp); + fclose(inpFp); + } + + // decompress + { + FILE* inpFp = fopen(lizardFilename, "rb"); + FILE* outFp = fopen(decFilename, "wb"); + + test_decompress(outFp, inpFp); + + fclose(outFp); + fclose(inpFp); + } + + // verify + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* decFp = fopen(decFilename, "rb"); + + const int cmp = compare(inpFp, decFp); + if(0 == cmp) { + printf("Verify : OK\n"); + } else { + printf("Verify : NG\n"); + } + + fclose(decFp); + fclose(inpFp); + } + + return 0; +} diff --git a/contrib/lizard/examples/compress_functions.c b/contrib/lizard/examples/compress_functions.c new file mode 100644 index 00000000000..36bc40e691a --- /dev/null +++ b/contrib/lizard/examples/compress_functions.c @@ -0,0 +1,303 @@ +/* + * compress_functions.c + * Copyright : Kyle Harper + * License : Follows same licensing as the lizard_compress.c/lizard_compress.h program at any given time. Currently, BSD 2. + * Description: A program to demonstrate the various compression functions involved in when using Lizard_compress_MinLevel(). The idea + * is to show how each step in the call stack can be used directly, if desired. There is also some benchmarking for + * each function to demonstrate the (probably lack of) performance difference when jumping the stack. + * (If you're new to lizard, please read simple_buffer.c to understand the fundamentals) + * + * The call stack (before theoretical compiler optimizations) for Lizard_compress_MinLevel is as follows: + * Lizard_compress_MinLevel + * Lizard_compress_fast + * Lizard_compress_extState_MinLevel + * Lizard_compress_generic + * + * Lizard_compress_MinLevel() + * This is the recommended function for compressing data. It will serve as the baseline for comparison. + * Lizard_compress_fast() + * Despite its name, it's not a "fast" version of compression. It simply decides if HEAPMODE is set and either + * allocates memory on the heap for a struct or creates the struct directly on the stack. Stack access is generally + * faster but this function itself isn't giving that advantage, it's just some logic for compile time. + * Lizard_compress_extState_MinLevel() + * This simply accepts all the pointers and values collected thus far and adds logic to determine how + * Lizard_compress_generic should be invoked; specifically: can the source fit into a single pass as determined by + * Lizard_64Klimit. + * Lizard_compress_generic() + * As the name suggests, this is the generic function that ultimately does most of the heavy lifting. Calling this + * directly can help avoid some test cases and branching which might be useful in some implementation-specific + * situations, but you really need to know what you're doing AND what you're asking lizard to do! You also need a + * wrapper function because this function isn't exposed with lizard_compress.h. + * + * The call stack for decompression functions is shallow. There are 2 options: + * Lizard_decompress_safe || Lizard_decompress_fast + * Lizard_decompress_generic + * + * Lizard_decompress_safe + * This is the recommended function for decompressing data. It is considered safe because the caller specifies + * both the size of the compresssed buffer to read as well as the maximum size of the output (decompressed) buffer + * instead of just the latter. + * Lizard_decompress_generic + * This is the generic function that both of the Lizard_decompress_* functions above end up calling. Calling this + * directly is not advised, period. Furthermore, it is a static inline function in lizard_compress.c, so there isn't a symbol + * exposed for anyone using lizard_compress.h to utilize. + * + * Special Note About Decompression: + * Using the Lizard_decompress_safe() function protects against malicious (user) input. + */ + +/* Since lizard compiles with c99 and not gnu/std99 we need to enable POSIX linking for time.h structs and functions. */ +#if __STDC_VERSION__ >= 199901L +#define _XOPEN_SOURCE 600 +#else +#define _XOPEN_SOURCE 500 +#endif +#define _POSIX_C_SOURCE 199309L + +/* Includes, for Power! */ +#include "lizard_compress.h" +#include "lizard_decompress.h" +#include /* for printf() */ +#include /* for exit() */ +#include /* for atoi() memcmp() */ +#include /* for uint_types */ +#include /* for PRIu64 */ +#include /* for clock_gettime() */ +#include /* for setlocale() */ + +/* We need to know what one billion is for clock timing. */ +#define BILLION 1000000000L + +/* Create a crude set of test IDs so we can switch on them later (Can't switch() on a char[] or char*). */ +#define ID__LIZARD_COMPRESS_DEFAULT 1 +#define ID__LIZARD_COMPRESS_GENERIC 4 +#define ID__LIZARD_DECOMPRESS_SAFE 5 + + + +/* + * Easy show-error-and-bail function. + */ +void run_screaming(const char *message, const int code) { + printf("%s\n", message); + exit(code); + return; +} + + +/* + * Centralize the usage function to keep main cleaner. + */ +void usage(const char *message) { + printf("Usage: ./argPerformanceTesting \n"); + run_screaming(message, 1); + return; +} + + + +/* + * Runs the benchmark for Lizard_compress_* based on function_id. + */ +uint64_t bench( + const char *known_good_dst, + const int function_id, + const int iterations, + const char *src, + char *dst, + const size_t src_size, + const size_t max_dst_size, + const size_t comp_size + ) { + uint64_t time_taken = 0; + int rv = 0; + const int warm_up = 5000; + struct timespec start, end; + Lizard_stream_t* state = Lizard_createStream_MinLevel(); + if (!state) return; + + // Select the right function to perform the benchmark on. We perform 5000 initial loops to warm the cache and ensure that dst + // remains matching to known_good_dst between successive calls. + switch(function_id) { + case ID__LIZARD_COMPRESS_DEFAULT: + printf("Starting benchmark for function: Lizard_compress_MinLevel()\n"); + for(int junk=0; junk 1) + iterations = atoi(argv[1]); + if (iterations < 1) + usage("Argument 1 (iterations) must be > 0."); + + // First we will create 2 sources (char *) of 2000 bytes each. One normal text, the other highly-compressible text. + const char *src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed luctus purus et risus vulputate, et mollis orci ullamcorper. Nulla facilisi. Fusce in ligula sed purus varius aliquet interdum vitae justo. Proin quis diam velit. Nulla varius iaculis auctor. Cras volutpat, justo eu dictum pulvinar, elit sem porttitor metus, et imperdiet metus sapien et ante. Nullam nisi nulla, ornare eu tristique eu, dignissim vitae diam. Nulla sagittis porta libero, a accumsan felis sagittis scelerisque. Integer laoreet eleifend congue. Etiam rhoncus leo vel dolor fermentum, quis luctus nisl iaculis. Praesent a erat sapien. Aliquam semper mi in lorem ultrices ultricies. Lorem ipsum dolor sit amet, consectetur adipiscing elit. In feugiat risus sed enim ultrices, at sodales nulla tristique. Maecenas eget pellentesque justo, sed pellentesque lectus. Fusce sagittis sit amet elit vel varius. Donec sed ligula nec ligula vulputate rutrum sed ut lectus. Etiam congue pharetra leo vitae cursus. Morbi enim ante, porttitor ut varius vel, tincidunt quis justo. Nunc iaculis, risus id ultrices semper, metus est efficitur ligula, vel posuere risus nunc eget purus. Ut lorem turpis, condimentum at sem sed, porta aliquam turpis. In ut sapien a nulla dictum tincidunt quis sit amet lorem. Fusce at est egestas, luctus neque eu, consectetur tortor. Phasellus eleifend ultricies nulla ac lobortis. Morbi maximus quam cursus vehicula iaculis. Maecenas cursus vel justo ut rutrum. Curabitur magna orci, dignissim eget dapibus vitae, finibus id lacus. Praesent rhoncus mattis augue vitae bibendum. Praesent porta mauris non ultrices fermentum. Quisque vulputate ipsum in sodales pulvinar. Aliquam nec mollis felis. Donec vitae augue pulvinar, congue nisl sed, pretium purus. Fusce lobortis mi ac neque scelerisque semper. Pellentesque vel est vitae magna aliquet aliquet. Nam non dolor. Nulla facilisi. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Morbi ac lacinia felis metus."; + const char *hc_src = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + // Set and derive sizes. Since we're using strings, use strlen() + 1 for \0. + const size_t src_size = strlen(src) + 1; + const size_t max_dst_size = Lizard_compressBound(src_size); + int bytes_returned = 0; + // Now build allocations for the data we'll be playing with. + char *dst = calloc(1, max_dst_size); + char *known_good_dst = calloc(1, max_dst_size); + char *known_good_hc_dst = calloc(1, max_dst_size); + if (dst == NULL || known_good_dst == NULL || known_good_hc_dst == NULL) + run_screaming("Couldn't allocate memory for the destination buffers. Sad :(", 1); + + // Create known-good buffers to verify our tests with other functions will produce the same results. + bytes_returned = Lizard_compress_MinLevel(src, known_good_dst, src_size, max_dst_size); + if (bytes_returned < 1) + run_screaming("Couldn't create a known-good destination buffer for comparison... this is bad.", 1); + const size_t src_comp_size = bytes_returned; + bytes_returned = Lizard_compress_MinLevel(hc_src, known_good_hc_dst, src_size, max_dst_size); + if (bytes_returned < 1) + run_screaming("Couldn't create a known-good (highly compressible) destination buffer for comparison... this is bad.", 1); + const size_t hc_src_comp_size = bytes_returned; + + + /* Lizard_compress_MinLevel() */ + // This is the default function so we don't need to demonstrate how to use it. See basics.c if you need more basal information. + + /* Lizard_compress_extState_MinLevel() */ + // Using this function directly requires that we build an Lizard_stream_t struct ourselves. We do NOT have to reset it ourselves. + memset(dst, 0, max_dst_size); + Lizard_stream_t* state = Lizard_createStream_MinLevel(); + if (!state) return; + bytes_returned = Lizard_compress_extState_MinLevel(state, src, dst, src_size, max_dst_size, 1); + if (bytes_returned < 1) + run_screaming("Failed to compress src using Lizard_compress_extState_MinLevel. echo $? for return code.", bytes_returned); + if (memcmp(dst, known_good_dst, bytes_returned) != 0) + run_screaming("According to memcmp(), the value we got in dst from Lizard_compress_extState_MinLevel doesn't match the known-good value. This is bad.", 1); + + /* Lizard_compress_generic */ + // When you can exactly control the inputs and options of your Lizard needs, you can use Lizard_compress_generic and fixed (const) + // values for the enum types such as dictionary and limitations. Any other direct-use is probably a bad idea. + // + // That said, the Lizard_compress_generic() function is 'static inline' and does not have a prototype in lizard_compress.h to expose a symbol + // for it. In other words: we can't access it directly. I don't want to submit a PR that modifies lizard_compress.c/h. Yann and others can + // do that if they feel it's worth expanding this example. + // + // I will, however, leave a skeleton of what would be required to use it directly: + /* + memset(dst, 0, max_dst_size); + // Lizard_stream_t state: is already declared above. We can reuse it BUT we have to reset the stream ourselves between each call. + Lizard_resetStream_MinLevel((Lizard_stream_t *)state); + // Since src size is small we know the following enums will be used: notLimited (0), byU16 (2), noDict (0), noDictIssue (0). + bytes_returned = Lizard_compress_generic(state, src, dst, src_size, max_dst_size, notLimited, byU16, noDict, noDictIssue, 1); + if (bytes_returned < 1) + run_screaming("Failed to compress src using Lizard_compress_generic. echo $? for return code.", bytes_returned); + if (memcmp(dst, known_good_dst, bytes_returned) != 0) + run_screaming("According to memcmp(), the value we got in dst from Lizard_compress_generic doesn't match the known-good value. This is bad.", 1); + */ + Lizard_freeStream(state); + + + /* Benchmarking */ + /* Now we'll run a few rudimentary benchmarks with each function to demonstrate differences in speed based on the function used. + * Remember, we cannot call Lizard_compress_generic() directly (yet) so it's disabled. + */ + // Suite A - Normal Compressibility + char *dst_d = calloc(1, src_size); + memset(dst, 0, max_dst_size); + printf("\nStarting suite A: Normal compressible text.\n"); + uint64_t time_taken__default = bench(known_good_dst, ID__LIZARD_COMPRESS_DEFAULT, iterations, src, dst, src_size, max_dst_size, src_comp_size); + //uint64_t time_taken__generic = bench(known_good_dst, ID__LIZARD_COMPRESS_GENERIC, iterations, src, dst, src_size, max_dst_size, src_comp_size); + uint64_t time_taken__decomp_safe = bench(src, ID__LIZARD_DECOMPRESS_SAFE, iterations, known_good_dst, dst_d, src_size, max_dst_size, src_comp_size); + // Suite B - Highly Compressible + memset(dst, 0, max_dst_size); + printf("\nStarting suite B: Highly compressible text.\n"); + uint64_t time_taken_hc__default = bench(known_good_hc_dst, ID__LIZARD_COMPRESS_DEFAULT, iterations, hc_src, dst, src_size, max_dst_size, hc_src_comp_size); + //uint64_t time_taken_hc__generic = bench(known_good_hc_dst, ID__LIZARD_COMPRESS_GENERIC, iterations, hc_src, dst, src_size, max_dst_size, hc_src_comp_size); + uint64_t time_taken_hc__decomp_safe = bench(hc_src, ID__LIZARD_DECOMPRESS_SAFE, iterations, known_good_hc_dst, dst_d, src_size, max_dst_size, hc_src_comp_size); + + // Report and leave. + setlocale(LC_ALL, ""); + const char *format = "|%-14s|%-30s|%'14.9f|%'16d|%'14d|%'13.2f%%|\n"; + const char *header_format = "|%-14s|%-30s|%14s|%16s|%14s|%14s|\n"; + const char *separator = "+--------------+------------------------------+--------------+----------------+--------------+--------------+\n"; + printf("\n"); + printf("%s", separator); + printf(header_format, "Source", "Function Benchmarked", "Total Seconds", "Iterations/sec", "ns/Iteration", "% of default"); + printf("%s", separator); + printf(format, "Normal Text", "Lizard_compress_MinLevel()", (double)time_taken__default / BILLION, (int)(iterations / ((double)time_taken__default /BILLION)), time_taken__default / iterations, (double)time_taken__default * 100 / time_taken__default); + printf(format, "Normal Text", "Lizard_compress_fast()", (double)time_taken__fast / BILLION, (int)(iterations / ((double)time_taken__fast /BILLION)), time_taken__fast / iterations, (double)time_taken__fast * 100 / time_taken__default); + printf(format, "Normal Text", "Lizard_compress_extState_MinLevel()", (double)time_taken__fast_extstate / BILLION, (int)(iterations / ((double)time_taken__fast_extstate /BILLION)), time_taken__fast_extstate / iterations, (double)time_taken__fast_extstate * 100 / time_taken__default); + //printf(format, "Normal Text", "Lizard_compress_generic()", (double)time_taken__generic / BILLION, (int)(iterations / ((double)time_taken__generic /BILLION)), time_taken__generic / iterations, (double)time_taken__generic * 100 / time_taken__default); + printf(format, "Normal Text", "Lizard_decompress_safe()", (double)time_taken__decomp_safe / BILLION, (int)(iterations / ((double)time_taken__decomp_safe /BILLION)), time_taken__decomp_safe / iterations, (double)time_taken__decomp_safe * 100 / time_taken__default); + printf(header_format, "", "", "", "", "", ""); + printf(format, "Compressible", "Lizard_compress_MinLevel()", (double)time_taken_hc__default / BILLION, (int)(iterations / ((double)time_taken_hc__default /BILLION)), time_taken_hc__default / iterations, (double)time_taken_hc__default * 100 / time_taken_hc__default); + printf(format, "Compressible", "Lizard_compress_fast()", (double)time_taken_hc__fast / BILLION, (int)(iterations / ((double)time_taken_hc__fast /BILLION)), time_taken_hc__fast / iterations, (double)time_taken_hc__fast * 100 / time_taken_hc__default); + printf(format, "Compressible", "Lizard_compress_extState_MinLevel()", (double)time_taken_hc__fast_extstate / BILLION, (int)(iterations / ((double)time_taken_hc__fast_extstate /BILLION)), time_taken_hc__fast_extstate / iterations, (double)time_taken_hc__fast_extstate * 100 / time_taken_hc__default); + //printf(format, "Compressible", "Lizard_compress_generic()", (double)time_taken_hc__generic / BILLION, (int)(iterations / ((double)time_taken_hc__generic /BILLION)), time_taken_hc__generic / iterations, (double)time_taken_hc__generic * 100 / time_taken_hc__default); + printf(format, "Compressible", "Lizard_decompress_safe()", (double)time_taken_hc__decomp_safe / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_safe /BILLION)), time_taken_hc__decomp_safe / iterations, (double)time_taken_hc__decomp_safe * 100 / time_taken_hc__default); + printf("%s", separator); + printf("\n"); + printf("All done, ran %d iterations per test.\n", iterations); + return 0; +} diff --git a/contrib/lizard/examples/frameCompress.c b/contrib/lizard/examples/frameCompress.c new file mode 100644 index 00000000000..37ea0aa4fa8 --- /dev/null +++ b/contrib/lizard/examples/frameCompress.c @@ -0,0 +1,169 @@ +// Lizardframe API example : compress a file +// Based on sample code from Zbigniew Jędrzejewski-Szmek + +#include +#include +#include +#include + +#include + +#define BUF_SIZE (16*1024) +#define LIZARD_HEADER_SIZE 19 +#define LIZARD_FOOTER_SIZE 4 + +static const LizardF_preferences_t lizard_preferences = { + { LizardF_max256KB, LizardF_blockLinked, LizardF_noContentChecksum, LizardF_frame, 0, { 0, 0 } }, + 0, /* compression level */ + 0, /* autoflush */ + { 0, 0, 0, 0 }, /* reserved, must be set to 0 */ +}; + +static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) { + LizardF_errorCode_t r; + LizardF_compressionContext_t ctx; + char *src, *buf = NULL; + size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size; + + r = LizardF_createCompressionContext(&ctx, LIZARDF_VERSION); + if (LizardF_isError(r)) { + printf("Failed to create context: error %zu", r); + return 1; + } + r = 1; + + src = malloc(BUF_SIZE); + if (!src) { + printf("Not enough memory"); + goto cleanup; + } + + frame_size = LizardF_compressBound(BUF_SIZE, &lizard_preferences); + size = frame_size + LIZARD_HEADER_SIZE + LIZARD_FOOTER_SIZE; + buf = malloc(size); + if (!buf) { + printf("Not enough memory"); + goto cleanup; + } + + n = offset = count_out = LizardF_compressBegin(ctx, buf, size, &lizard_preferences); + if (LizardF_isError(n)) { + printf("Failed to start compression: error %zu", n); + goto cleanup; + } + + printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n); + + for (;;) { + k = fread(src, 1, BUF_SIZE, in); + if (k == 0) + break; + count_in += k; + + n = LizardF_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL); + if (LizardF_isError(n)) { + printf("Compression failed: error %zu", n); + goto cleanup; + } + + offset += n; + count_out += n; + if (size - offset < frame_size + LIZARD_FOOTER_SIZE) { + printf("Writing %zu bytes\n", offset); + + k = fwrite(buf, 1, offset, out); + if (k < offset) { + if (ferror(out)) + printf("Write failed"); + else + printf("Short write"); + goto cleanup; + } + + offset = 0; + } + } + + n = LizardF_compressEnd(ctx, buf + offset, size - offset, NULL); + if (LizardF_isError(n)) { + printf("Failed to end compression: error %zu", n); + goto cleanup; + } + + offset += n; + count_out += n; + printf("Writing %zu bytes\n", offset); + + k = fwrite(buf, 1, offset, out); + if (k < offset) { + if (ferror(out)) + printf("Write failed"); + else + printf("Short write"); + goto cleanup; + } + + *size_in = count_in; + *size_out = count_out; + r = 0; + cleanup: + if (ctx) + LizardF_freeCompressionContext(ctx); + free(src); + free(buf); + return r; +} + +static int compress(const char *input, const char *output) { + char *tmp = NULL; + FILE *in = NULL, *out = NULL; + size_t size_in = 0, size_out = 0; + int r = 1; + + if (!output) { + size_t len = strlen(input); + + output = tmp = malloc(len + 5); + if (!tmp) { + printf("Not enough memory"); + return 1; + } + strcpy(tmp, input); + strcpy(tmp + len, ".liz"); + } + + in = fopen(input, "rb"); + if (!in) { + fprintf(stderr, "Failed to open input file %s: %s\n", input, strerror(errno)); + goto cleanup; + } + + out = fopen(output, "wb"); + if (!out) { + fprintf(stderr, "Failed to open output file %s: %s\n", output, strerror(errno)); + goto cleanup; + } + + r = compress_file(in, out, &size_in, &size_out); + if (r == 0) + printf("%s: %zu → %zu bytes, %.1f%%\n", + input, size_in, size_out, + (double)size_out / size_in * 100); + cleanup: + if (in) + fclose(in); + if (out) + fclose(out); + free(tmp); + return r; +} + + +int main(int argc, char **argv) { + if (argc < 2 || argc > 3) { + fprintf(stderr, "Syntax: %s \n", argv[0]); + return EXIT_FAILURE; + } + + return compress(argv[1], argv[2]); +} diff --git a/contrib/lizard/examples/printVersion.c b/contrib/lizard/examples/printVersion.c new file mode 100644 index 00000000000..9818fdd98cf --- /dev/null +++ b/contrib/lizard/examples/printVersion.c @@ -0,0 +1,13 @@ +// Lizard trivial example : print Library version number +// Copyright : Takayuki Matsuoka & Yann Collet + + +#include +#include "lizard_compress.h" + +int main(int argc, char** argv) +{ + (void)argc; (void)argv; + printf("Hello World ! Lizard Library version = %d\n", Lizard_versionNumber()); + return 0; +} diff --git a/contrib/lizard/examples/simple_buffer.c b/contrib/lizard/examples/simple_buffer.c new file mode 100644 index 00000000000..154d3b2cb76 --- /dev/null +++ b/contrib/lizard/examples/simple_buffer.c @@ -0,0 +1,91 @@ +/* + * simple_buffer.c + * Copyright : Kyle Harper + * License : Follows same licensing as the lizard_compress.c/lizard_compress.h program at any given time. Currently, BSD 2. + * Description: Example program to demonstrate the basic usage of the compress/decompress functions within lizard_compress.c/lizard_compress.h. + * The functions you'll likely want are Lizard_compress_MinLevel and Lizard_decompress_safe. Both of these are documented in + * the lizard_compress.h header file; I recommend reading them. + */ + +/* Includes, for Power! */ +#include "lizard_compress.h" // This is all that is required to expose the prototypes for basic compression and decompression. +#include "lizard_decompress.h" +#include // For printf() +#include // For memcmp() +#include // For exit() + +/* + * Easy show-error-and-bail function. + */ +void run_screaming(const char *message, const int code) { + printf("%s\n", message); + exit(code); + return; +} + + +/* + * main + */ +int main(void) { + /* Introduction */ + // Below we will have a Compression and Decompression section to demonstrate. There are a few important notes before we start: + // 1) The return codes of Lizard_ functions are important. Read lizard_compress.h if you're unsure what a given code means. + // 2) Lizard uses char* pointers in all Lizard_ functions. This is baked into the API and probably not going to change. If your + // program uses pointers that are unsigned char*, void*, or otherwise different you may need to do some casting or set the + // right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign). + + /* Compression */ + // We'll store some text into a variable pointed to by *src to be compressed later. + const char *src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; + // The compression function needs to know how many bytes of exist. Since we're using a string, we can use strlen() + 1 (for \0). + const size_t src_size = strlen(src) + 1; + // Lizard provides a function that will tell you the maximum size of compressed output based on input data via Lizard_compressBound(). + const size_t max_dst_size = Lizard_compressBound(src_size); + // We will use that size for our destination boundary when allocating space. + char *compressed_data = malloc(max_dst_size); + if (compressed_data == NULL) + run_screaming("Failed to allocate memory for *compressed_data.", 1); + // That's all the information and preparation Lizard needs to compress *src into *compressed_data. Invoke Lizard_compress_MinLevel now + // with our size values and pointers to our memory locations. Save the return value for error checking. + int return_value = 0; + return_value = Lizard_compress_MinLevel(src, compressed_data, src_size, max_dst_size); + // Check return_value to determine what happened. + if (return_value < 0) + run_screaming("A negative result from Lizard_compress_MinLevel indicates a failure trying to compress the data. See exit code (echo $?) for value returned.", return_value); + if (return_value == 0) + run_screaming("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.", 1); + if (return_value > 0) + printf("We successfully compressed some data!\n"); + // Not only does a positive return_value mean success, the value returned == the number of bytes required. You can use this to + // realloc() *compress_data to free up memory, if desired. We'll do so just to demonstrate the concept. + const size_t compressed_data_size = return_value; + compressed_data = (char *)realloc(compressed_data, compressed_data_size); + if (compressed_data == NULL) + run_screaming("Failed to re-alloc memory for compressed_data. Sad :(", 1); + + /* Decompression */ + // Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite! We'll create a + // *new_src location of size src_size since we know that value. + char *new_src = malloc(src_size); + if (new_src == NULL) + run_screaming("Failed to allocate memory for *new_src.", 1); + // The Lizard_decompress_safe function needs to know where the compressed data is, how many bytes long it is, where the new_src + // memory location is, and how large the new_src (uncompressed) output will be. Again, save the return_value. + return_value = Lizard_decompress_safe(compressed_data, new_src, compressed_data_size, src_size); + if (return_value < 0) + run_screaming("A negative result from Lizard_decompress_fast indicates a failure trying to decompress the data. See exit code (echo $?) for value returned.", return_value); + if (return_value == 0) + run_screaming("I'm not sure this function can ever return 0. Documentation in lizard_compress.h doesn't indicate so.", 1); + if (return_value > 0) + printf("We successfully decompressed some data!\n"); + // Not only does a positive return value mean success, the value returned == the number of bytes read from the compressed_data + // stream. I'm not sure there's ever a time you'll need to know this in most cases... + + /* Validation */ + // We should be able to compare our original *src with our *new_src and be byte-for-byte identical. + if (memcmp(src, new_src, src_size) != 0) + run_screaming("Validation failed. *src and *new_src are not identical.", 1); + printf("Validation done. The string we ended up with is:\n%s\n", new_src); + return 0; +} diff --git a/contrib/lizard/examples/streaming_api_basics.md b/contrib/lizard/examples/streaming_api_basics.md new file mode 100644 index 00000000000..85c06803455 --- /dev/null +++ b/contrib/lizard/examples/streaming_api_basics.md @@ -0,0 +1,87 @@ +# Lizard Streaming API Basics +by *Takayuki Matsuoka* +## Lizard API sets + +Lizard has the following API sets : + + - "Auto Framing" API (lizard_frame.h) : + This is most recommended API for usual application. + It guarantees interoperability with other Lizard framing format compliant tools/libraries + such as Lizard command line utility, node-lizard, etc. + - "Block" API : This is recommended for simple purpose. + It compress single raw memory block to Lizard memory block and vice versa. + - "Streaming" API : This is designed for complex thing. + For example, compress huge stream data in restricted memory environment. + +Basically, you should use "Auto Framing" API. +But if you want to write advanced application, it's time to use Block or Streaming APIs. + + +## What is difference between Block and Streaming API ? + +Block API (de)compresses single contiguous memory block. +In other words, Lizard library find redundancy from single contiguous memory block. +Streaming API does same thing but (de)compress multiple adjacent contiguous memory block. +So Lizard library could find more redundancy than Block API. + +The following figure shows difference between API and block sizes. +In these figures, original data is splitted to 4KiBytes contiguous chunks. + +``` +Original Data + +---------------+---------------+----+----+----+ + | 4KiB Chunk A | 4KiB Chunk B | C | D |... | + +---------------+---------------+----+----+----+ + +Example (1) : Block API, 4KiB Block + +---------------+---------------+----+----+----+ + | 4KiB Chunk A | 4KiB Chunk B | C | D |... | + +---------------+---------------+----+----+----+ + | Block #1 | Block #2 | #3 | #4 |... | + +---------------+---------------+----+----+----+ + + (No Dependency) + + +Example (2) : Block API, 8KiB Block + +---------------+---------------+----+----+----+ + | 4KiB Chunk A | 4KiB Chunk B | C | D |... | + +---------------+---------------+----+----+----+ + | Block #1 |Block #2 |... | + +--------------------+----------+-------+-+----+ + ^ | ^ | + | | | | + +--------------+ +----+ + Internal Dependency Internal Dependency + + +Example (3) : Streaming API, 4KiB Block + +---------------+---------------+-----+----+----+ + | 4KiB Chunk A | 4KiB Chunk B | C | D |... | + +---------------+---------------+-----+----+----+ + | Block #1 | Block #2 | #3 | #4 |... | + +---------------+----+----------+-+---+-+--+----+ + ^ | ^ | ^ | + | | | | | | + +--------------+ +--------+ +---+ + Dependency Dependency Dependency +``` + + - In example (1), there is no dependency. + All blocks are compressed independently. + - In example (2), naturally 8KiBytes block has internal dependency. + But still block #1 and #2 are compressed independently. + - In example (3), block #2 has dependency to #1, + also #3 has dependency to #2 and #1, #4 has #3, #2 and #1, and so on. + +Here, we can observe difference between example (2) and (3). +In (2), there's no dependency between chunk B and C, but (3) has dependency between B and C. +This dependency improves compression ratio. + + +## Restriction of Streaming API + +For the efficiency, Streaming API doesn't keep mirror copy of dependent (de)compressed memory. +This means users should keep these dependent (de)compressed memory explicitly. +Usually, "Dependent memory" is previous adjacent contiguous memory up to 64KiBytes. +Lizard will not access further memories. diff --git a/contrib/lizard/lib/.gitignore b/contrib/lizard/lib/.gitignore new file mode 100644 index 00000000000..6eedf69343a --- /dev/null +++ b/contrib/lizard/lib/.gitignore @@ -0,0 +1,3 @@ +# make install artefact +liblizard.pc +test/ diff --git a/contrib/lizard/lib/LICENSE b/contrib/lizard/lib/LICENSE new file mode 100644 index 00000000000..b335b0bafcf --- /dev/null +++ b/contrib/lizard/lib/LICENSE @@ -0,0 +1,25 @@ +Lizard Library +Copyright (C) 2011-2016, Yann Collet. +Copyright (C) 2016-2017, Przemyslaw Skibinski +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/lizard/lib/README.md b/contrib/lizard/lib/README.md new file mode 100644 index 00000000000..2775b984d63 --- /dev/null +++ b/contrib/lizard/lib/README.md @@ -0,0 +1,76 @@ +Lizard - Library Files +================================ + +The __lib__ directory contains several directories. +Depending on target use case, it's enough to include only files from relevant directories. + + +#### API + +Lizard stable API is exposed within [lizard_compress.h](lizard_compress.h) and [lizard_decompress.h](lizard_decompress.h), +at the root of `lib` directory. + + +#### Compatibility issues + +The raw Lizard block compression format is detailed within [lizard_Block_format]. +To compress an arbitrarily long file or data stream, multiple blocks are required. +Organizing these blocks and providing a common header format to handle their content +is the purpose of the Frame format, defined in [lizard_Frame_format]. +`lizard` command line utility produces files or streams compatible with the Frame format. +(_Advanced stuff_ : It's possible to hide xxhash symbols into a local namespace. +This is what `liblizard` does, to avoid symbol duplication +in case a user program would link to several libraries containing xxhash symbols.) + +[lizard_Block_format]: ../doc/lizard_Block_format.md +[lizard_Frame_format]: ../doc/lizard_Frame_format.md + + +#### Various Lizard builds + +Files `lizard_common.h`, `lizard_compress*`, `lizard_parser_*.h`, `lizard_decompress*`, and `entropy\mem.h` are required in all circumstances. + +To compile: +- Lizard_raw only with levels 10...29 : use the `-DLIZARD_NO_HUFFMAN` compiler flag +- Lizard_raw with levels 10...49 : include also all files from `entropy` directory +- Lizard_frame with levels 10...49 : `lizard_frame*` and all files from `entropy` and `xxhash` directories + + +#### Advanced API + +A more complex `lizard_frame_static.h` is also provided. +It contains definitions which are not guaranteed to remain stable within future versions. +It must be used with static linking ***only***. + + +#### Using MinGW+MSYS to create DLL + +DLL can be created using MinGW+MSYS with the `make liblizard` command. +This command creates `dll\liblizard.dll` and the import library `dll\liblizard.lib`. +The import library is only required with Visual C++. +The header files `lizard.h`, `lizardhc.h`, `lizard_frame.h` and the dynamic library +`dll\liblizard.dll` are required to compile a project using gcc/MinGW. +The dynamic library has to be added to linking options. +It means that if a project that uses Lizard consists of a single `test-dll.c` +file it should be compiled with `liblizard.lib`. For example: +``` + gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblizard.dll +``` +The compiled executable will require Lizard DLL which is available at `dll\liblizard.dll`. + + +#### Miscellaneous + +Other files present in the directory are not source code. There are : + + - LICENSE : contains the BSD license text + - Makefile : script to compile or install lizard library (static or dynamic) + - liblizard.pc.in : for pkg-config (make install) + - README.md : this file + + +#### License + +All source material within __lib__ directory are BSD 2-Clause licensed. +See [LICENSE](LICENSE) for details. +The license is also repeated at the top of each source file. diff --git a/contrib/lizard/lib/dll/liblizard.def b/contrib/lizard/lib/dll/liblizard.def new file mode 100644 index 00000000000..73ebc44570a --- /dev/null +++ b/contrib/lizard/lib/dll/liblizard.def @@ -0,0 +1,19 @@ +LIBRARY liblizard.dll +EXPORTS + Lizard_compress + Lizard_compressBound + Lizard_compress_continue + Lizard_compress_extState + Lizard_createStream + Lizard_createStreamDecode + Lizard_decompress_safe + Lizard_decompress_safe_continue + Lizard_decompress_safe_partial + Lizard_decompress_safe_usingDict + Lizard_freeStream + Lizard_freeStreamDecode + Lizard_loadDict + Lizard_resetStream + Lizard_saveDict + Lizard_setStreamDecode + Lizard_sizeofState diff --git a/contrib/lizard/lib/entropy/README.md b/contrib/lizard/lib/entropy/README.md new file mode 100644 index 00000000000..5b18f20b063 --- /dev/null +++ b/contrib/lizard/lib/entropy/README.md @@ -0,0 +1,38 @@ +New Generation Entropy library +============================== + +The __lib__ directory contains several files, but you don't necessarily want them all. +Here is a detailed list, to help you decide which one you need : + + +#### Compulsory files + +These files are required in all circumstances : +- __error_public.h__ : error list as enum +- __error_private.h__ : error management +- __mem.h__ : low level memory access routines +- __bitstream.h__ : generic read/write bitstream common to all entropy codecs +- __entropy_common.c__ : common functions needed for both compression and decompression + + +#### Finite State Entropy + +This is the base codec required by other ones. +It implements a tANS variant, similar to arithmetic in compression performance, but much faster. Compression and decompression can be compiled independently. +- __fse.h__ : exposes interfaces +- __fse_compress.c__ : implements compression codec +- __fse_decompress.c__ : implements decompression codec + + +#### FSE 16-bits symbols version + +This codec is able to encode alphabets of size > 256, using 2 bytes per symbol. It requires the base FSE codec to compile properly. Compression and decompression are merged in the same file. +- __fseU16.c__ implements the codec, while __fseU16.h__ exposes its interfaces. + + +#### Huffman codec + +This is the fast huffman codec. It requires the base FSE codec to compress its headers. Compression and decompression can be compiled independently. +- __huf.h__ : exposes interfaces. +- __huf_compress.c__ : implements compression codec +- __huf_decompress.c__ : implements decompression codec diff --git a/contrib/lizard/lib/entropy/bitstream.h b/contrib/lizard/lib/entropy/bitstream.h new file mode 100644 index 00000000000..849a9e6018a --- /dev/null +++ b/contrib/lizard/lib/entropy/bitstream.h @@ -0,0 +1,414 @@ +/* ****************************************************************** + bitstream + Part of FSE library + header file (to include) + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + +/*-**************************************** +* Dependencies +******************************************/ +#include "mem.h" /* unaligned access routines */ +#include "error_private.h" /* error codes and messages */ + + +/*========================================= +* Target specific +=========================================*/ +#if defined(__BMI__) && defined(__GNUC__) +# include /* support for bextr (experimental) */ +#endif + + +/*-****************************************** +* bitStream encoding API (write forward) +********************************************/ +/* bitStream can mix input from multiple sources. +* A critical property of these streams is that they encode and decode in **reverse** direction. +* So the first bit sequence you add will be the last to be read, like a LIFO stack. +*/ +typedef struct +{ + size_t bitContainer; + int bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. +* bitStream will never write outside of this buffer. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* +* bits are first added to a local register. +* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Writing data into memory is an explicit operation, performed by the flushBits function. +* Hence keep track how many bits are potentially stored into local register to avoid register overflow. +* After a flushBits, a maximum of 7 bits might still be stored into local register. +* +* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. +* +* Last operation is to close the bitStream. +* The function returns the final size of CStream in bytes. +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) +*/ + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef struct +{ + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + + + +/*-************************************************************** +* Internal functions +****************************************************************/ +MEM_STATIC unsigned BIT_highbit32 (register U32 val) +{ +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + _BitScanReverse ( &r, val ); + return (unsigned) r; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ + return 31 - __builtin_clz (val); +# else /* Software version */ + static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; +# endif +} + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ + + +/*-************************************************************** +* bitStream encoding +****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(void*) + * @return : 0 if success, + otherwise an error code (can be tested using ERR_isError() ) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr); + if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall); + return 0; +} + +/*! BIT_addBits() : + can add up to 26 bits into `bitC`. + Does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) +{ + bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits) +{ + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ +} + +/*! BIT_flushBits() : + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + + if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */ + + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : +* Initialize a BIT_DStream_t. +* `bitD` : a pointer to an already allocated BIT_DStream_t structure. +* `srcSize` must be the *exact* size of the bitStream, in bytes. +* @return : size of stream (== srcSize) or an errorCode if a problem is detected +*/ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->start = (const char*)srcBuffer; + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->start = (const char*)srcBuffer; + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); // fallthrough + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); // fallthrough + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); // fallthrough + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; // fallthrough + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; // fallthrough + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; // fallthrough + default:; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ +#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008 /* experimental */ +# if defined(__x86_64__) + if (sizeof(bitContainer)==8) + return _bextr_u64(bitContainer, start, nbBits); + else +# endif + return _bextr_u32(bitContainer, start, nbBits); +#else + return (bitContainer >> start) & BIT_mask[nbBits]; +#endif +} + +MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ + return bitContainer & BIT_mask[nbBits]; +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted + */ + MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ +#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); +#endif +} + +/*! BIT_lookBitsFast() : +* unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; + return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); +} + +MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. + */ +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : +* unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStream() : +* Refill `bitD` from buffer previously set in BIT_initDStream() . +* This function is safe, it guarantees it will not read beyond src buffer. +* @return : status of `BIT_DStream_t` internal register. + if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ + return result; + } +} + +/*! BIT_endOfDStream() : +* @return Tells if DStream has exactly reached its end (all bits consumed). +*/ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ diff --git a/contrib/lizard/lib/entropy/entropy_common.c b/contrib/lizard/lib/entropy/entropy_common.c new file mode 100644 index 00000000000..72bc398da38 --- /dev/null +++ b/contrib/lizard/lib/entropy/entropy_common.c @@ -0,0 +1,231 @@ +/* + Common functions of New Generation Entropy library + Copyright (C) 2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +*************************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ +#include "huf.h" + + +/*-**************************************** +* Version +******************************************/ +unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } + + +/*-**************************************** +* FSE Error Management +******************************************/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } + +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/* ************************************************************** +* HUF Error Management +****************************************************************/ +unsigned HUF_isError(size_t code) { return ERR_isError(code); } + +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + if (hbSize < 4) return ERROR(srcSize_wrong); + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<1) & (charnum<=*maxSVPtr)) { + if (previous0) { + unsigned n0 = charnum; + while ((bitStream & 0xFFFF) == 0xFFFF) { + n0 += 24; + if (ip < iend-5) { + ip += 2; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 16; + bitCount += 16; + } } + while ((bitStream & 3) == 3) { + n0 += 3; + bitStream >>= 2; + bitCount += 2; + } + n0 += bitStream & 3; + bitCount += 2; + if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); + while (charnum < n0) normalizedCounter[charnum++] = 0; + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 2; + } } + { int const max = (2*threshold-1) - remaining; + int count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = bitStream & (threshold-1); + bitCount += nbBits-1; + } else { + count = bitStream & (2*threshold-1); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + remaining -= count < 0 ? -count : count; /* -1 means +1 */ + normalizedCounter[charnum++] = (short)count; + previous0 = !count; + while (remaining < threshold) { + nbBits--; + threshold >>= 1; + } + + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> (bitCount & 31); + } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ + if (remaining != 1) return ERROR(corruption_detected); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */ + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n= HUF_TABLELOG_MAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} diff --git a/contrib/lizard/lib/entropy/error_private.h b/contrib/lizard/lib/entropy/error_private.h new file mode 100644 index 00000000000..1340c16bf25 --- /dev/null +++ b/contrib/lizard/lib/entropy/error_private.h @@ -0,0 +1,115 @@ +/* ****************************************************************** + Error codes and messages + Copyright (C) 2013-2016, Yann Collet + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Homepage : http://www.zstd.net +****************************************************************** */ +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* Dependencies +******************************************/ +#include /* size_t */ +#include "error_public.h" /* enum list */ + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef FSE_ErrorCode ERR_enum; +#define PREFIX(name) FSE_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#ifdef ERROR +# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#endif +#define ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + + +/*-**************************************** +* Error Strings +******************************************/ + +ERR_STATIC const char* ERR_getErrorString(ERR_enum code) +{ + static const char* notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size incorrect"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(maxCode): + default: return notErrorCode; + } +} + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ diff --git a/contrib/lizard/lib/entropy/error_public.h b/contrib/lizard/lib/entropy/error_public.h new file mode 100644 index 00000000000..a8524719889 --- /dev/null +++ b/contrib/lizard/lib/entropy/error_public.h @@ -0,0 +1,64 @@ +/* ****************************************************************** + Error codes list + Copyright (C) 2016, Yann Collet + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ +#ifndef ERROR_PUBLIC_H_MODULE +#define ERROR_PUBLIC_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* error codes list +******************************************/ +typedef enum { + FSE_error_no_error, + FSE_error_GENERIC, + FSE_error_dstSize_tooSmall, + FSE_error_srcSize_wrong, + FSE_error_corruption_detected, + FSE_error_tableLog_tooLarge, + FSE_error_maxSymbolValue_tooLarge, + FSE_error_maxSymbolValue_tooSmall, + FSE_error_maxCode +} FSE_ErrorCode; + +/* note : compare with size_t function results using FSE_getError() */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_PUBLIC_H_MODULE */ diff --git a/contrib/lizard/lib/entropy/fse.h b/contrib/lizard/lib/entropy/fse.h new file mode 100644 index 00000000000..baac3903267 --- /dev/null +++ b/contrib/lizard/lib/entropy/fse.h @@ -0,0 +1,694 @@ +/* ****************************************************************** + FSE : Finite State Entropy codec + Public Prototypes declaration + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef FSE_H +#define FSE_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +/*-***************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ + + +/*-***************************************** +* FSE_PUBLIC_API : control library symbols visibility +******************************************/ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define FSE_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define FSE_PUBLIC_API +#endif + +/*------ Version ------*/ +#define FSE_VERSION_MAJOR 0 +#define FSE_VERSION_MINOR 9 +#define FSE_VERSION_RELEASE 0 + +#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE +#define FSE_QUOTE(str) #str +#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) +#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) + +#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) +FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ + +/*-**************************************** +* FSE simple functions +******************************************/ +/*! FSE_compress() : + Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. + 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). + @return : size of compressed data (<= dstCapacity). + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. + if FSE_isError(return), compression failed (more details using FSE_getErrorName()) +*/ +FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/*! FSE_decompress(): + Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', + into already allocated destination buffer 'dst', of size 'dstCapacity'. + @return : size of regenerated data (<= maxDstSize), + or an error code, which can be tested using FSE_isError() . + + ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! + Why ? : making this distinction requires a header. + Header management is intentionally delegated to the user layer, which can better manage special cases. +*/ +FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize); + + +/*-***************************************** +* Tool functions +******************************************/ +FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE advanced functions +******************************************/ +/*! FSE_compress2() : + Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' + Both parameters can be defined as '0' to mean : use default value + @return : size of compressed data + Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. + if FSE_isError(return), it's an error code. +*/ +FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_count(): + Provides the precise count of each byte within a table 'count'. + 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). + *maxSymbolValuePtr will be updated if detected smaller than initial value. + @return : the count of the most frequent symbol (which is not identified). + if return == srcSize, there is only one symbol. + Can also return an error code, which can be tested with FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ +FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue); +FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); + +/*! Constructor and Destructor of FSE_DTable. + Note that its size depends on 'tableLog' */ +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ +FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); +FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); + +/*! FSE_buildDTable(): + Builds 'dt', which must be already allocated, using FSE_createDTable(). + return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_decompress_usingDTable(): + Decompress compressed source `cSrc` of size `cSrcSize` using `dt` + into `dst` which must be already allocated. + @return : size of regenerated data (necessarily <= `dstCapacity`), + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) +*/ + + +#ifdef FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) (size + (size>>7)) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<= `1024` unsigned + */ +size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, unsigned* workSpace); + +/** FSE_countFast() : + * same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr + */ +size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + +/* FSE_countFast_wksp() : + * Same as FSE_countFast(), but using an externally provided scratch buffer. + * `workSpace` must be a table of minimum `1024` unsigned + */ +size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace); + +/*! FSE_count_simple + * Same as FSE_countFast(), but does not use any additional memory (not even on stack). + * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`). +*/ +size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + + + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ + +/* FSE_compress_wksp() : + * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). + * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. + */ +#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + (1<<((maxTableLog>2)?(maxTableLog-2):0)) ) +size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); +/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ + +size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` must be >= `(1<= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; + statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol) +{ + const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* FSE_H */ diff --git a/contrib/lizard/lib/entropy/fse_compress.c b/contrib/lizard/lib/entropy/fse_compress.c new file mode 100644 index 00000000000..337b7a6ffac --- /dev/null +++ b/contrib/lizard/lib/entropy/fse_compress.c @@ -0,0 +1,848 @@ +/* ****************************************************************** + FSE : Finite State Entropy encoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include /* printf (debug) */ +#include "bitstream.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + U32 const step = FSE_TABLESTEP(tableSize); + U32 cumul[FSE_MAX_SYMBOL_VALUE+2]; + + FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace; + U32 highThreshold = tableSize-1; + + /* CTable header */ + if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge); + tableU16[-2] = (U16) tableLog; + tableU16[-1] = (U16) maxSymbolValue; + + /* For explanations on how to distribute symbol values over the table : + * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + + /* symbol start positions */ + { U32 u; + cumul[0] = 0; + for (u=1; u<=maxSymbolValue+1; u++) { + if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ + cumul[u] = cumul[u-1] + 1; + tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); + } else { + cumul[u] = cumul[u-1] + normalizedCounter[u-1]; + } } + cumul[maxSymbolValue+1] = tableSize+1; + } + + /* Spread symbols */ + { U32 position = 0; + U32 symbol; + for (symbol=0; symbol<=maxSymbolValue; symbol++) { + int nbOccurences; + for (nbOccurences=0; nbOccurences highThreshold) position = (position + step) & tableMask; /* Low proba area */ + } } + + if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ + } + + /* Build table */ + { U32 u; for (u=0; u> 3) + 3; + return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ +} + +static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) +{ + BYTE* const ostart = (BYTE*) header; + BYTE* out = ostart; + BYTE* const oend = ostart + headerBufferSize; + int nbBits; + const int tableSize = 1 << tableLog; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + bitStream = 0; + bitCount = 0; + /* Table Size */ + bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; + bitCount += 4; + + /* Init */ + remaining = tableSize+1; /* +1 for extra accuracy */ + threshold = tableSize; + nbBits = tableLog+1; + + while (remaining>1) { /* stops at 1 */ + if (previous0) { + unsigned start = charnum; + while (!normalizedCounter[charnum]) charnum++; + while (charnum >= start+24) { + start+=24; + bitStream += 0xFFFFU << bitCount; + if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE) bitStream; + out[1] = (BYTE)(bitStream>>8); + out+=2; + bitStream>>=16; + } + while (charnum >= start+3) { + start+=3; + bitStream += 3 << bitCount; + bitCount += 2; + } + bitStream += (charnum-start) << bitCount; + bitCount += 2; + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + { int count = normalizedCounter[charnum++]; + int const max = (2*threshold-1)-remaining; + remaining -= count < 0 ? -count : count; + count++; /* +1 for extra accuracy */ + if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + bitStream += count << bitCount; + bitCount += nbBits; + bitCount -= (count>=1; + } + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + + /* flush remaining bitStream */ + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out+= (bitCount+7) /8; + + if (charnum > maxSymbolValue + 1) return ERROR(GENERIC); + + return (out-ostart); +} + + +size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); +} + + + +/*-************************************************************** +* Counting histogram +****************************************************************/ +/*! FSE_count_simple + This function counts byte values within `src`, and store the histogram into table `count`. + It doesn't use any additional memory. + But this function is unsafe : it doesn't check that all values within `src` can fit into `count`. + For this reason, prefer using a table `count` with 256 elements. + @return : count of most numerous element +*/ +size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned max=0; + + memset(count, 0, (maxSymbolValue+1)*sizeof(*count)); + if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } + + while (ip max) max = count[s]; } + + return (size_t)max; +} + + +/* FSE_count_parallel_wksp() : + * Same as FSE_count_parallel(), but using an externally provided scratch buffer. + * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */ +static size_t FSE_count_parallel_wksp( + unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + unsigned checkMax, unsigned* const workSpace) +{ + const BYTE* ip = (const BYTE*)source; + const BYTE* const iend = ip+sourceSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned max=0; + U32* const Counting1 = workSpace; + U32* const Counting2 = Counting1 + 256; + U32* const Counting3 = Counting2 + 256; + U32* const Counting4 = Counting3 + 256; + + memset(Counting1, 0, 4*256*sizeof(unsigned)); + + /* safety checks */ + if (!sourceSize) { + memset(count, 0, maxSymbolValue + 1); + *maxSymbolValuePtr = 0; + return 0; + } + if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */ + + /* by stripes of 16 bytes */ + { U32 cached = MEM_read32(ip); ip += 4; + while (ip < iend-15) { + U32 c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + } + ip-=4; + } + + /* finish last symbols */ + while (ipmaxSymbolValue; s--) { + Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; + if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall); + } } + + { U32 s; for (s=0; s<=maxSymbolValue; s++) { + count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; + if (count[s] > max) max = count[s]; + } } + + while (!count[maxSymbolValue]) maxSymbolValue--; + *maxSymbolValuePtr = maxSymbolValue; + return (size_t)max; +} + +/* FSE_countFast_wksp() : + * Same as FSE_countFast(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= `1024` unsigned */ +size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, unsigned* workSpace) +{ + if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); + return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace); +} + +/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ +size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + unsigned tmpCounters[1024]; + return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters); +} + +/* FSE_count_wksp() : + * Same as FSE_count(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= `1024` unsigned */ +size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, unsigned* workSpace) +{ + if (*maxSymbolValuePtr < 255) + return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace); + *maxSymbolValuePtr = 255; + return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace); +} + +size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + unsigned tmpCounters[1024]; + return FSE_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters); +} + + + +/*-************************************************************** +* FSE Compression Code +****************************************************************/ +/*! FSE_sizeof_CTable() : + FSE_CTable is a variable size structure which contains : + `U16 tableLog;` + `U16 maxSymbolValue;` + `U16 nextStateNumber[1 << tableLog];` // This size is variable + `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable +Allocation is manual (C standard does not support variable-size structures). +*/ +size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + return FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); +} + +FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) +{ + size_t size; + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); + return (FSE_CTable*)malloc(size); +} + +void FSE_freeCTable (FSE_CTable* ct) { free(ct); } + +/* provides the minimum logSize to safely represent a distribution */ +static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) +{ + U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; + U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + return minBits; +} + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) +{ + U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; + U32 tableLog = maxTableLog; + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; + if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; + return tableLog; +} + +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); +} + + +/* Secondary normalization method. + To be used when primary method fails. */ + +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue) +{ + U32 s; + U32 distributed = 0; + U32 ToDistribute; + + /* Init */ + U32 const lowThreshold = (U32)(total >> tableLog); + U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == 0) { + norm[s]=0; + continue; + } + if (count[s] <= lowThreshold) { + norm[s] = -1; + distributed++; + total -= count[s]; + continue; + } + if (count[s] <= lowOne) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + norm[s]=-2; + } + ToDistribute = (1 << tableLog) - distributed; + + if ((total / ToDistribute) > lowOne) { + /* risk of rounding to zero */ + lowOne = (U32)((total * 3) / (ToDistribute * 2)); + for (s=0; s<=maxSymbolValue; s++) { + if ((norm[s] == -2) && (count[s] <= lowOne)) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } } + ToDistribute = (1 << tableLog) - distributed; + } + + if (distributed == maxSymbolValue+1) { + /* all values are pretty poor; + probably incompressible data (should have already been detected); + find max, then give all remaining points to max */ + U32 maxV = 0, maxC = 0; + for (s=0; s<=maxSymbolValue; s++) + if (count[s] > maxC) maxV=s, maxC=count[s]; + norm[maxV] += (short)ToDistribute; + return 0; + } + + { U64 const vStepLog = 62 - tableLog; + U64 const mid = (1ULL << (vStepLog-1)) - 1; + U64 const rStep = ((((U64)1<> vStepLog); + U32 const sEnd = (U32)(end >> vStepLog); + U32 const weight = sEnd - sStart; + if (weight < 1) + return ERROR(GENERIC); + norm[s] = (short)weight; + tmpTotal = end; + } } } + + return 0; +} + + +size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t total, + unsigned maxSymbolValue) +{ + /* Sanity checks */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ + if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ + + { U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + U64 const scale = 62 - tableLog; + U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */ + U64 const vStep = 1ULL<<(scale-20); + int stillToDistribute = 1<> tableLog); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == total) return 0; /* rle special case */ + if (count[s] == 0) { normalizedCounter[s]=0; continue; } + if (count[s] <= lowThreshold) { + normalizedCounter[s] = -1; + stillToDistribute--; + } else { + short proba = (short)((count[s]*step) >> scale); + if (proba<8) { + U64 restToBeat = vStep * rtbTable[proba]; + proba += (count[s]*step) - ((U64)proba< restToBeat; + } + if (proba > largestP) largestP=proba, largest=s; + normalizedCounter[s] = proba; + stillToDistribute -= proba; + } } + if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { + /* corner case, need another normalization method */ + size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); + if (FSE_isError(errorCode)) return errorCode; + } + else normalizedCounter[largest] += (short)stillToDistribute; + } + +#if 0 + { /* Print Table (debug) */ + U32 s; + U32 nTotal = 0; + for (s=0; s<=maxSymbolValue; s++) + printf("%3i: %4i \n", s, normalizedCounter[s]); + for (s=0; s<=maxSymbolValue; s++) + nTotal += abs(normalizedCounter[s]); + if (nTotal != (1U<>1); /* assumption : tableLog >= 1 */ + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* header */ + tableU16[-2] = (U16) nbBits; + tableU16[-1] = (U16) maxSymbolValue; + + /* Build table */ + for (s=0; s FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } + + /* 2 or 4 encoding per loop */ + while ( ip>istart ) { + + FSE_encodeSymbol(&bitC, &CState2, *--ip); + + if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ + FSE_FLUSHBITS(&bitC); + + FSE_encodeSymbol(&bitC, &CState1, *--ip); + + if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + } + + FSE_FLUSHBITS(&bitC); + } + + FSE_flushCState(&bitC, &CState2); + FSE_flushCState(&bitC, &CState1); + return BIT_closeCStream(&bitC); +} + +size_t FSE_compress_usingCTable (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct) +{ + unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); + + if (fast) + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); + else + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); +} + + +size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } + +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + +/* FSE_compress_wksp() : + * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` size must be `(1< not compressible */ + if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ + } + + tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) ); + + /* Write table description header */ + { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); + op += nc_err; + } + + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } + + /* check compressibility */ + if ( (size_t)(op-ostart) >= srcSize-1 ) return 0; + + return op-ostart; +} + +typedef struct { + FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; + BYTE scratchBuffer[1 << FSE_MAX_TABLELOG]; +} fseWkspMax_t; + +size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) +{ + fseWkspMax_t scratchBuffer; + FSE_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer)); +} + +size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); +} + + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/contrib/lizard/lib/entropy/fse_decompress.c b/contrib/lizard/lib/entropy/fse_decompress.c new file mode 100644 index 00000000000..1479a5e8217 --- /dev/null +++ b/contrib/lizard/lib/entropy/fse_decompress.c @@ -0,0 +1,329 @@ +/* ****************************************************************** + FSE : Finite State Entropy decoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include /* printf (debug) */ +#include "bitstream.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + +/* check and forward error code */ +#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ +FSE_DTable* FSE_createDTable (unsigned tableLog) +{ + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); +} + +void FSE_freeDTable (FSE_DTable* dt) +{ + free(dt); +} + +size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + { U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; utableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSV1 = tableMask+1; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + + +size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + short counting[FSE_MAX_SYMBOL_VALUE+1]; + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + + /* normal FSE decoding mode */ + size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + if (FSE_isError(NCountLength)) return NCountLength; + //if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */ + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + ip += NCountLength; + cSrcSize -= NCountLength; + + CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) ); + + return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */ +} + + +typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; + +size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) +{ + DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ + return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG); +} + + + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/contrib/lizard/lib/entropy/huf.h b/contrib/lizard/lib/entropy/huf.h new file mode 100644 index 00000000000..48c7f831f55 --- /dev/null +++ b/contrib/lizard/lib/entropy/huf.h @@ -0,0 +1,250 @@ +/* ****************************************************************** + Huffman coder, part of New Generation Entropy library + header file + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* *** Dependencies *** */ +#include /* size_t */ + + +/*-*** PUBLIC_API : control library symbols visibility *** */ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define HUF_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define HUF_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define HUF_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define HUF_PUBLIC_API +#endif + + +/* *** simple functions *** */ +/** +HUF_compress() : + Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. + 'dst' buffer must be already allocated. + Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). + `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. + @return : size of compressed data (<= `dstCapacity`). + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single repeated byte symbol (RLE compression). + if HUF_isError(return), compression failed (more details using HUF_getErrorName()) +*/ +HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/** +HUF_decompress() : + Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', + into already allocated buffer 'dst', of minimum size 'dstSize'. + `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. + Note : in contrast with FSE, HUF_decompress can regenerate + RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, + because it knows size to regenerate. + @return : size of regenerated data (== originalSize), + or an error code, which can be tested using HUF_isError() +*/ +HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize); + + +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +/* *** Advanced function *** */ + +/** HUF_compress2() : + * Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` . + * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ +HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + +/** HUF_compress4X_wksp() : +* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */ +HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least 1024 unsigned */ + + + +#ifdef HUF_STATIC_LINKING_ONLY + +/* *** Dependencies *** */ +#include "mem.h" /* U32 */ + + +/* *** Constants *** */ +#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ +#define HUF_SYMBOLVALUE_MAX 255 +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + U32 name##hb[maxSymbolValue+1]; \ + void* name##hv = &(name##hb); \ + HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } +#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ +size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ +size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + + +/* **************************************** +* HUF detailed API +******************************************/ +/*! +HUF_compress() does the following: +1. count symbol occurrence from source[] into table count[] using FSE_count() +2. (optional) refine tableLog using HUF_optimalTableLog() +3. build Huffman table from count using HUF_buildCTable() +4. save Huffman table to memory buffer using HUF_writeCTable() +5. encode the data stream using HUF_compress4X_usingCTable() + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and regenerate 'CTable' using external methods. +*/ +/* FSE_count() : find it within "fse.h" */ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); +typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ +size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); +size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. + */ +size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize); + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/** HUF_readCTable() : +* Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); + + +/* +HUF_decompress() does the following: +1. select the decompression algorithm (X2, X4) based on pre-computed heuristics +2. build Huffman table from save, using HUF_readDTableXn() +3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable +*/ + +/** HUF_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize); + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + + +/* single stream variants */ + +size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least 1024 unsigned */ +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ + +size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); +size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ +size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + +#endif /* HUF_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* HUF_H_298734234 */ diff --git a/contrib/lizard/lib/entropy/huf_compress.c b/contrib/lizard/lib/entropy/huf_compress.c new file mode 100644 index 00000000000..a47a1a2c761 --- /dev/null +++ b/contrib/lizard/lib/entropy/huf_compress.c @@ -0,0 +1,612 @@ +/* ****************************************************************** + Huffman encoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include /* memcpy, memset */ +#include /* printf (debug) */ +#include "bitstream.h" +#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ +#include "fse.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + + +/* ************************************************************** +* Utils +****************************************************************/ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); +} + + +/* ******************************************************* +* HUF : Huffman block compression +*********************************************************/ +/* HUF_compressWeights() : + * Same as FSE_compress(), but dedicated to huff0's weights compression. + * The use case needs much less stack memory. + * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. + */ +#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 +size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const oend = ostart + dstSize; + + U32 maxSymbolValue = HUF_TABLELOG_MAX; + U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; + + FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; + BYTE scratchBuffer[1< not compressible */ + } + + tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) ); + + /* Write table description header */ + { CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); + op += hSize; + } + + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } + + return op-ostart; +} + + +struct HUF_CElt_s { + U16 val; + BYTE nbBits; +}; /* typedef'd to HUF_CElt within "huf.h" */ + +/*! HUF_writeCTable() : + `CTable` : huffman tree to save, using huf representation. + @return : size of saved CTable */ +size_t HUF_writeCTable (void* dst, size_t maxDstSize, + const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog) +{ + BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; + BYTE* op = (BYTE*)dst; + U32 n; + + /* check conditions */ + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + + /* convert to weight */ + bitsToWeight[0] = 0; + for (n=1; n1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ + op[0] = (BYTE)hSize; + return hSize+1; + } } + + /* write raw values as 4-bits (max : 15) */ + if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ + if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); + huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ + for (n=0; n HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall); + + /* Prepare base value per rank */ + { U32 n, nextRankStart = 0; + for (n=1; n<=tableLog; n++) { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = current; + } } + + /* fill nbBits */ + { U32 n; for (n=0; nn=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; + { U32 n; for (n=0; n0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + /* assign value within rank, symbol order */ + { U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; } + } + + return readSize; +} + + +typedef struct nodeElt_s { + U32 count; + U16 parent; + BYTE byte; + BYTE nbBits; +} nodeElt; + +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) +{ + const U32 largestBits = huffNode[lastNonNull].nbBits; + if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */ + + /* there are several too large elements (at least >= 2) */ + { int totalCost = 0; + const U32 baseCost = 1 << (largestBits - maxNbBits); + U32 n = lastNonNull; + + while (huffNode[n].nbBits > maxNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)maxNbBits; + n --; + } /* n stops at huffNode[n].nbBits <= maxNbBits */ + while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ + + /* renorm totalCost */ + totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ + + /* repay normalized cost */ + { U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_TABLELOG_MAX+2]; + int pos; + + /* Get pos of last (smallest) symbol per rank */ + memset(rankLast, 0xF0, sizeof(rankLast)); + { U32 currentNbBits = maxNbBits; + for (pos=n ; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) continue; + currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ + rankLast[maxNbBits-currentNbBits] = pos; + } } + + while (totalCost > 0) { + U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; + for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 highPos = rankLast[nBitsToDecrease]; + U32 lowPos = rankLast[nBitsToDecrease-1]; + if (highPos == noSymbol) continue; + if (lowPos == noSymbol) break; + { U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) break; + } } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + nBitsToDecrease ++; + totalCost -= 1 << (nBitsToDecrease-1); + if (rankLast[nBitsToDecrease-1] == noSymbol) + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ + huffNode[rankLast[nBitsToDecrease]].nbBits ++; + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } } /* while (totalCost > 0) */ + + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ + while (huffNode[n].nbBits == maxNbBits) n--; + huffNode[n+1].nbBits--; + rankLast[1] = n+1; + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } } } /* there are several too large elements (at least >= 2) */ + + return maxNbBits; +} + + +typedef struct { + U32 base; + U32 current; +} rankPos; + +static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue) +{ + rankPos rank[32]; + U32 n; + + memset(rank, 0, sizeof(rank)); + for (n=0; n<=maxSymbolValue; n++) { + U32 r = BIT_highbit32(count[n] + 1); + rank[r].base ++; + } + for (n=30; n>0; n--) rank[n-1].base += rank[n].base; + for (n=0; n<32; n++) rank[n].current = rank[n].base; + for (n=0; n<=maxSymbolValue; n++) { + U32 const c = count[n]; + U32 const r = BIT_highbit32(c+1) + 1; + U32 pos = rank[r].current++; + while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--; + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } +} + + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. + */ +#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) +typedef nodeElt huffNodeTable[2*HUF_SYMBOLVALUE_MAX+1 +1]; +size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize) +{ + nodeElt* const huffNode0 = (nodeElt*)workSpace; + nodeElt* const huffNode = huffNode0+1; + U32 n, nonNullRank; + int lowS, lowN; + U16 nodeNb = STARTNODE; + U32 nodeRoot; + + /* safety checks */ + if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); /* workSpace is not large enough */ + if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC); + memset(huffNode0, 0, sizeof(huffNodeTable)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue); + + /* init for parents */ + nonNullRank = maxSymbolValue; + while(huffNode[nonNullRank].count == 0) nonNullRank--; + lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; + huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb; + nodeNb++; lowS-=2; + for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); + huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ + + /* create parents */ + while (nodeNb <= nodeRoot) { + U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = nodeNb; + nodeNb++; + } + + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n=nodeRoot-1; n>=STARTNODE; n--) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + for (n=0; n<=nonNullRank; n++) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + + /* enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); + + /* fill result into tree (val, nbBits) */ + { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine stating value per rank */ + { U16 min = 0; + for (n=maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; n<=maxSymbolValue; n++) + tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ + for (n=0; n<=maxSymbolValue; n++) + tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ + } + + return maxNbBits; +} + +/** HUF_buildCTable() : + * Note : count is used before tree is written, so they can safely overlap + */ +size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) +{ + huffNodeTable nodeTable; + return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable)); +} + +static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) +{ + BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); +} + +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } + +#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) + +#define HUF_FLUSHBITS_1(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream) + +#define HUF_FLUSHBITS_2(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream) + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + const BYTE* ip = (const BYTE*) src; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + size_t n; + const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize)); + BIT_CStream_t bitC; + + /* init */ + if (dstSize < 8) return 0; /* not enough space to compress */ + { size_t const initErr = BIT_initCStream(&bitC, op, oend-op); + if (HUF_isError(initErr)) return 0; } + + n = srcSize & ~3; /* join to mod 4 */ + switch (srcSize & 3) + { + case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); + HUF_FLUSHBITS_2(&bitC); + // fallthrough + case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); + HUF_FLUSHBITS_1(&bitC); + // fallthrough + case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable); + HUF_FLUSHBITS(&bitC); + // fallthrough + case 0 : + default: ; + } + + for (; n>0; n-=4) { /* note : n&3==0 at this stage */ + HUF_encodeSymbol(&bitC, ip[n- 1], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 2], CTable); + HUF_FLUSHBITS_2(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 3], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 4], CTable); + HUF_FLUSHBITS(&bitC); + } + + return BIT_closeCStream(&bitC); +} + + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ + if (srcSize < 12) return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); + if (cSize==0) return 0; + MEM_writeLE16(ostart, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); + if (cSize==0) return 0; + MEM_writeLE16(ostart+2, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); + if (cSize==0) return 0; + MEM_writeLE16(ostart+4, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) ); + if (cSize==0) return 0; + op += cSize; + } + + return op-ostart; +} + + +/* `workSpace` must a table of at least 1024 unsigned */ +static size_t HUF_compress_internal ( + void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + unsigned singleStream, + void* workSpace, size_t wkspSize) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + union { + U32 count[HUF_SYMBOLVALUE_MAX+1]; + HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1]; + } table; /* `count` can overlap with `CTable`; saves 1 KB */ + + /* checks & inits */ + if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); + if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */ + if (!dstSize) return 0; /* cannot fit within dst budget */ + if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; + if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; + + /* Scan input and build symbol stats */ + { CHECK_V_F(largest, FSE_count_wksp (table.count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) ); + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ + if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ + } + + /* Build Huffman Tree */ + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + { CHECK_V_F(maxBits, HUF_buildCTable_wksp (table.CTable, table.count, maxSymbolValue, huffLog, workSpace, wkspSize) ); + huffLog = (U32)maxBits; + } + + /* Write table description header */ + { CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, table.CTable, maxSymbolValue, huffLog) ); + if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */ + op += hSize; + } + + /* Compress */ + { size_t const cSize = (singleStream) ? + HUF_compress1X_usingCTable(op, oend - op, src, srcSize, table.CTable) : /* single segment */ + HUF_compress4X_usingCTable(op, oend - op, src, srcSize, table.CTable); + if (HUF_isError(cSize)) return cSize; + if (cSize==0) return 0; /* uncompressible */ + op += cSize; + } + + /* check compressibility */ + if ((size_t)(op-ostart) >= srcSize-1) + return 0; + + return op-ostart; +} + + +size_t HUF_compress1X_wksp (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize); +} + +size_t HUF_compress1X (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + unsigned workSpace[1024]; + return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); +} + +size_t HUF_compress4X_wksp (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize); +} + +size_t HUF_compress2 (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + unsigned workSpace[1024]; + return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); +} + +size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT); +} diff --git a/contrib/lizard/lib/entropy/huf_decompress.c b/contrib/lizard/lib/entropy/huf_decompress.c new file mode 100644 index 00000000000..a342dfb1e4a --- /dev/null +++ b/contrib/lizard/lib/entropy/huf_decompress.c @@ -0,0 +1,885 @@ +/* ****************************************************************** + Huffman decoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +/* inline is defined */ +#elif defined(_MSC_VER) || defined(__GNUC__) +# define inline __inline +#else +# define inline /* disable inline */ +#endif + +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/* ************************************************************** +* Dependencies +****************************************************************/ +#include /* memcpy, memset */ +#include "bitstream.h" /* BIT_* */ +#include "fse.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ + +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ + +typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ + +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize) +{ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + + HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) return iSize; + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Prepare ranks */ + { U32 n, nextRankStart = 0; + for (n=1; n> 1; + U32 i; + HUF_DEltX2 D; + D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); + for (i = rankVal[w]; i < rankVal[w] + length; i++) + dt[i] = D; + rankVal[w] += length; + } } + + return iSize; +} + + +static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + + /* closer to the end */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, hence no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + return pEnd-pStart; +} + +static size_t HUF_decompress1X2_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; } + + HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +size_t HUF_decompress1X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + +size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2 (DCtx, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); +} + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); +} + + +static size_t HUF_decompress4X2_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable + 1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; } + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endSignal) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; + } +} + + +size_t HUF_decompress4X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + + +size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2 (dctx, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx); +} + +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ + +typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; + +/* HUF_fillDTableX4Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, + const U32* rankValOrigin, const int minWeight, + const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, + U32 nbBitsBaseline, U16 baseSeq) +{ + HUF_DEltX4 DElt; + U32 rankVal[HUF_TABLELOG_MAX + 1]; + + /* get pre-calculated rankVal */ + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill skipped values */ + if (minWeight>1) { + U32 i, skipSize = rankVal[minWeight]; + MEM_writeLE16(&(DElt.sequence), baseSeq); + DElt.nbBits = (BYTE)(consumed); + DElt.length = 1; + for (i = 0; i < skipSize; i++) + DTable[i] = DElt; + } + + /* fill DTable */ + { U32 s; for (s=0; s= 1 */ + + rankVal[weight] += length; + } } +} + +typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1]; + +static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, const U32 sortedListSize, + const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32 rankVal[HUF_TABLELOG_MAX + 1]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + U32 s; + + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill DTable */ + for (s=0; s= minBits) { /* enough room for a second symbol */ + U32 sortedRank; + int minWeight = nbBits + scaleLog; + if (minWeight < 1) minWeight = 1; + sortedRank = rankStart[minWeight]; + HUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, + rankValOrigin[nbBits], minWeight, + sortedList+sortedRank, sortedListSize-sortedRank, + nbBitsBaseline, symbol); + } else { + HUF_DEltX4 DElt; + MEM_writeLE16(&(DElt.sequence), symbol); + DElt.nbBits = (BYTE)(nbBits); + DElt.length = 1; + { U32 const end = start + length; + U32 u; + for (u = start; u < end; u++) DTable[u] = DElt; + } } + rankVal[weight] += length; + } +} + +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) +{ + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + U32 rankStats[HUF_TABLELOG_MAX + 1] = { 0 }; + U32 rankStart0[HUF_TABLELOG_MAX + 2] = { 0 }; + U32* const rankStart = rankStart0+1; + rankVal_t rankVal; + U32 tableLog, maxW, sizeOfSort, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr; + + HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + + /* find maxWeight */ + for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; w> consumed; + } } } } + + HUF_fillDTableX4(dt, maxTableLog, + sortedSymbol, sizeOfSort, + rankStart0, rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 1); + if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); + else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + } } + return 1; +} + + +#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX4_2(p, bitDPtr); + HUF_DECODE_SYMBOLX4_1(p, bitDPtr); + HUF_DECODE_SYMBOLX4_2(p, bitDPtr); + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + + if (p < pEnd) + p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + + +static size_t HUF_decompress1X4_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; + } + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +size_t HUF_decompress1X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + +size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX4 (DCtx, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); +} + +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + +static size_t HUF_decompress4X4_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; + const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; } + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) { + HUF_DECODE_SYMBOLX4_2(op1, &bitD1); + HUF_DECODE_SYMBOLX4_2(op2, &bitD2); + HUF_DECODE_SYMBOLX4_2(op3, &bitD3); + HUF_DECODE_SYMBOLX4_2(op4, &bitD4); + HUF_DECODE_SYMBOLX4_1(op1, &bitD1); + HUF_DECODE_SYMBOLX4_1(op2, &bitD2); + HUF_DECODE_SYMBOLX4_1(op3, &bitD3); + HUF_DECODE_SYMBOLX4_1(op4, &bitD4); + HUF_DECODE_SYMBOLX4_2(op1, &bitD1); + HUF_DECODE_SYMBOLX4_2(op2, &bitD2); + HUF_DECODE_SYMBOLX4_2(op3, &bitD3); + HUF_DECODE_SYMBOLX4_2(op4, &bitD4); + HUF_DECODE_SYMBOLX4_0(op1, &bitD1); + HUF_DECODE_SYMBOLX4_0(op2, &bitD2); + HUF_DECODE_SYMBOLX4_0(op3, &bitD3); + HUF_DECODE_SYMBOLX4_0(op4, &bitD4); + + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + + +size_t HUF_decompress4X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + + +size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX4 (dctx, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); +} + +size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + + +/* ********************************/ +/* Generic decompression selector */ +/* ********************************/ + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); + return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : + HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); +} + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); + return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : + HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); +} + + +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ + {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ + {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ + {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ + {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ + {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ + {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ + {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ + {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ + {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ + {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ + {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ + {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ + {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ + {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ + {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ +}; + +/** HUF_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + /* decoder timing evaluation */ + U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ + + return DTime1 < DTime0; +} + + +typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); + +size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + static const decompressionAlgo decompress[2] = { HUF_decompress4X2, HUF_decompress4X4 }; + + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); + } +} + +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} + +size_t HUF_decompress4X_hufOnly (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} + +size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} diff --git a/contrib/lizard/lib/entropy/mem.h b/contrib/lizard/lib/entropy/mem.h new file mode 100644 index 00000000000..708d897a1cc --- /dev/null +++ b/contrib/lizard/lib/entropy/mem.h @@ -0,0 +1,372 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ +#include /* memcpy */ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif +#if defined(__GNUC__) +# define MEM_STATIC static __inline __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +/* code only tested on 32 and 64 bits systems */ +#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } +MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; + typedef intptr_t iPtrDiff; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; + typedef ptrdiff_t iPtrDiff; +#endif + + +/*-************************************************************** +* Memory I/O +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define MEM_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) + __pragma( pack(push, 1) ) + typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign; + __pragma( pack(pop) ) +#else + typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; +#endif + +MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } +MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +#endif +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403) + return __builtin_bswap64(in); +#else + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + + +/* function safe only for comparisons */ +MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ diff --git a/contrib/lizard/lib/liblizard.pc.in b/contrib/lizard/lib/liblizard.pc.in new file mode 100644 index 00000000000..233c2e08faf --- /dev/null +++ b/contrib/lizard/lib/liblizard.pc.in @@ -0,0 +1,15 @@ +# Lizard - Fast LZ compression algorithm +# Copyright (C) 2011-2014, Yann Collet +# Copyright (C) 2016-2017, Przemyslaw Skibinski +# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +prefix=@PREFIX@ +libdir=@LIBDIR@ +includedir=@INCLUDEDIR@ + +Name: lizard +Description: extremely fast lossless compression algorithm library +URL: http://github.com/inikep/lizard +Version: @VERSION@ +Libs: -L@LIBDIR@ -llizard +Cflags: -I@INCLUDEDIR@ diff --git a/contrib/lizard/lib/lizard_common.h b/contrib/lizard/lib/lizard_common.h new file mode 100644 index 00000000000..45730cf4e86 --- /dev/null +++ b/contrib/lizard/lib/lizard_common.h @@ -0,0 +1,504 @@ +/* + Lizard - Fast LZ compression algorithm + Copyright (C) 2011-2015, Yann Collet + Copyright (C) 2016-2017, Przemyslaw Skibinski + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + +#ifndef LIZARD_COMMON_H_2983 +#define LIZARD_COMMON_H_2983 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/*-************************************ +* Memory routines +**************************************/ +#include /* malloc, calloc, free */ +#include /* memset, memcpy */ +#include /* intptr_t */ +#include "entropy/mem.h" +#include "lizard_compress.h" /* LIZARD_GCC_VERSION */ + +//#define LIZARD_USE_LOGS +#define LIZARD_LOG_COMPRESS(...) //printf(__VA_ARGS__) +#define LIZARD_LOG_DECOMPRESS(...) //printf(__VA_ARGS__) + +#define LIZARD_LOG_COMPRESS_LZ4(...) //printf(__VA_ARGS__) +#define COMPLOG_CODEWORDS_LZ4(...) //printf(__VA_ARGS__) +#define LIZARD_LOG_DECOMPRESS_LZ4(...) //printf(__VA_ARGS__) +#define DECOMPLOG_CODEWORDS_LZ4(...) //printf(__VA_ARGS__) + +#define LIZARD_LOG_COMPRESS_LIZv1(...) //printf(__VA_ARGS__) +#define COMPLOG_CODEWORDS_LIZv1(...) //printf(__VA_ARGS__) +#define LIZARD_LOG_DECOMPRESS_LIZv1(...) //printf(__VA_ARGS__) +#define DECOMPLOG_CODEWORDS_LIZv1(...) //printf(__VA_ARGS__) + + + + +/*-************************************ +* Common Constants +**************************************/ +#define MINMATCH 4 +//#define USE_LZ4_ONLY +//#define LIZARD_USE_TEST + +#define LIZARD_DICT_SIZE (1<<24) +#define WILDCOPYLENGTH 16 +#define LASTLITERALS WILDCOPYLENGTH +#define MFLIMIT (WILDCOPYLENGTH+MINMATCH) + +#define LIZARD_MAX_PRICE (1<<28) +#define LIZARD_INIT_LAST_OFFSET 0 +#define LIZARD_MAX_16BIT_OFFSET (1<<16) +#define MM_LONGOFF 16 +#define LIZARD_BLOCK_SIZE_PAD (LIZARD_BLOCK_SIZE+32) +#define LIZARD_COMPRESS_ADD_BUF (5*LIZARD_BLOCK_SIZE_PAD) +#ifndef LIZARD_NO_HUFFMAN + #define LIZARD_COMPRESS_ADD_HUF HUF_compressBound(LIZARD_BLOCK_SIZE_PAD) + #define LIZARD_HUF_BLOCK_SIZE LIZARD_BLOCK_SIZE +#else + #define LIZARD_COMPRESS_ADD_HUF 0 + #define LIZARD_HUF_BLOCK_SIZE 1 +#endif + +/* LZ4 codewords */ +#define ML_BITS_LZ4 4 +#define ML_MASK_LZ4 ((1U<= 1<<16 */ + U32 sufficientLength; /* used only by optimal parser: size of matches which is acceptable: larger == more compression, slower */ + U32 fullSearch; /* used only by optimal parser: perform full search of matches: 1 == more compression, slower */ + Lizard_parser_type parserType; + Lizard_decompress_type decompressType; +} Lizard_parameters; + + +struct Lizard_stream_s +{ + const BYTE* end; /* next block here to continue on current prefix */ + const BYTE* base; /* All index relative to this position */ + const BYTE* dictBase; /* alternate base for extDict */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more dict */ + U32 nextToUpdate; /* index from which to continue dictionary update */ + U32 allocatedMemory; + int compressionLevel; + Lizard_parameters params; + U32 hashTableSize; + U32 chainTableSize; + U32* chainTable; + U32* hashTable; + int last_off; + const BYTE* off24pos; + U32 huffType; + U32 comprStreamLen; + + BYTE* huffBase; + BYTE* huffEnd; + BYTE* offset16Base; + BYTE* offset24Base; + BYTE* lenBase; + BYTE* literalsBase; + BYTE* flagsBase; + BYTE* offset16Ptr; + BYTE* offset24Ptr; + BYTE* lenPtr; + BYTE* literalsPtr; + BYTE* flagsPtr; + BYTE* offset16End; + BYTE* offset24End; + BYTE* lenEnd; + BYTE* literalsEnd; + BYTE* flagsEnd; + U32 flagFreq[256]; + U32 litFreq[256]; + U32 litSum, flagSum; + U32 litPriceSum, log2LitSum, log2FlagSum; + U32 cachedPrice; + U32 cachedLitLength; + const BYTE* cachedLiterals; + const BYTE* diffBase; + const BYTE* srcBase; + const BYTE* destBase; +}; + +struct Lizard_streamDecode_s { + const BYTE* externalDict; + size_t extDictSize; + const BYTE* prefixEnd; + size_t prefixSize; +}; + +struct Lizard_dstream_s +{ + const BYTE* offset16Ptr; + const BYTE* offset24Ptr; + const BYTE* lenPtr; + const BYTE* literalsPtr; + const BYTE* flagsPtr; + const BYTE* offset16End; + const BYTE* offset24End; + const BYTE* lenEnd; + const BYTE* literalsEnd; + const BYTE* flagsEnd; + const BYTE* diffBase; + intptr_t last_off; +}; + +typedef struct Lizard_dstream_s Lizard_dstream_t; + +/* ************************************* +* HC Pre-defined compression levels +***************************************/ +#define LIZARD_WINDOWLOG_LZ4 16 +#define LIZARD_CHAINLOG_LZ4 LIZARD_WINDOWLOG_LZ4 +#define LIZARD_HASHLOG_LZ4 18 +#define LIZARD_HASHLOG_LZ4SM 12 + +#define LIZARD_WINDOWLOG_LIZv1 22 +#define LIZARD_CHAINLOG_LIZv1 LIZARD_WINDOWLOG_LIZv1 +#define LIZARD_HASHLOG_LIZv1 18 + + + +static const Lizard_parameters Lizard_defaultParameters[LIZARD_MAX_CLEVEL+1-LIZARD_MIN_CLEVEL] = +{ + /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ + { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4SM, 0, 0, 0, 0, 0, 0, Lizard_parser_fastSmall, Lizard_coderwords_LZ4 }, // level 10 + { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_fast, Lizard_coderwords_LZ4 }, // level 11 + { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_noChain, Lizard_coderwords_LZ4 }, // level 12 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 2, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 13 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 4, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 14 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 8, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 15 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 16, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 16 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 256, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 17 + { LIZARD_WINDOWLOG_LZ4, LIZARD_WINDOWLOG_LZ4+1, LIZARD_HASHLOG_LZ4, 16, 16, 4, 0, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LZ4 }, // level 18 + { LIZARD_WINDOWLOG_LZ4, LIZARD_WINDOWLOG_LZ4+1, 23, 16, 256, 4, 0, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LZ4 }, // level 19 + /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ + { LIZARD_WINDOWLOG_LIZv1, 0, 14, 0, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_fastBig, Lizard_coderwords_LIZv1 }, // level 20 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 14, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 21 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 22 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 23 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 2, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 24 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 8, 4, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 25 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 8, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 26 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 128, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 27 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 28 + { 24, 25, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 29 +#ifndef LIZARD_NO_HUFFMAN + /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ + { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4SM, 0, 0, 0, 0, 0, 0, Lizard_parser_fastSmall, Lizard_coderwords_LZ4 }, // level 30 + { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_fast, Lizard_coderwords_LZ4 }, // level 31 + { LIZARD_WINDOWLOG_LZ4, 0, 14, 0, 0, 0, 0, 0, 0, Lizard_parser_noChain, Lizard_coderwords_LZ4 }, // level 32 + { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_noChain, Lizard_coderwords_LZ4 }, // level 33 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 2, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 34 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 4, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 35 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 8, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 36 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 16, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 37 + { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 256, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 38 + { LIZARD_WINDOWLOG_LZ4, LIZARD_WINDOWLOG_LZ4+1, 23, 16, 256, 4, 0, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LZ4 }, // level 39 + /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ + { LIZARD_WINDOWLOG_LIZv1, 0, 14, 0, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_fastBig, Lizard_coderwords_LIZv1 }, // level 40 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 14, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 41 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 42 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 43 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 2, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 44 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 8, 4, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 45 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 8, 4, MM_LONGOFF, 64, 0, Lizard_parser_optimalPrice, Lizard_coderwords_LIZv1 }, // level 46 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 8, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 47 + { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 128, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 48 + { 24, 25, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 49 +#endif +// { 10, 10, 10, 0, 0, 4, 0, 0, 0, Lizard_fast }, // min values +// { 24, 24, 28, 24, 1<<24, 7, 0, 1<<24, 2, Lizard_optimal_price }, // max values +}; + + + +/*-************************************ +* Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#else +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# if defined(__GNUC__) || defined(__clang__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif /* _MSC_VER */ + +#define LIZARD_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#if (LIZARD_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define ALLOCATOR(n,s) calloc(n,s) +#define FREEMEM free +#define MEM_INIT memset +#ifndef MAX + #define MAX(a,b) ((a)>(b))?(a):(b) +#endif +#ifndef MIN + #define MIN(a,b) ((a)<(b)?(a):(b)) +#endif + +#if MINMATCH == 3 + #define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr)<<8) +#else + #define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr)) +#endif + + + + +/*-************************************ +* Reading and writing into memory +**************************************/ +#define STEPSIZE sizeof(size_t) + + +MEM_STATIC void Lizard_copy8(void* dst, const void* src) +{ + memcpy(dst,src,8); +} + +/* customized variant of memcpy, which can overwrite up to 7 bytes beyond dstEnd */ +MEM_STATIC void Lizard_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + +#if 0 + const size_t l2 = 8 - (((size_t)d) & (sizeof(void*)-1)); + Lizard_copy8(d,s); if (d>e-9) return; + d+=l2; s+=l2; +#endif /* join to align */ + + do { Lizard_copy8(d,s); d+=8; s+=8; } while (d= 3) /* GCC Intrinsic */ + return 31 - __builtin_clz(val); +# else /* Software version */ + static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + int r; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; + return r; +# endif +} + + +/*-************************************ +* Common functions +**************************************/ +MEM_STATIC unsigned Lizard_NbCommonBytes (register size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LIZARD_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LIZARD_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else /* Big Endian CPU */ { + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LIZARD_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT) + return (__builtin_clzll((U64)val) >> 3); +# else + unsigned r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LIZARD_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } + } +} + +MEM_STATIC unsigned Lizard_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + while (likely(pIn + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOTLizard_hash4Ptr + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + + +/* ************************************* +* Includes +***************************************/ +#include "lizard_compress.h" +#include "lizard_common.h" +#include +#include // intptr_t +#ifndef USE_LZ4_ONLY + #ifdef LIZARD_USE_TEST + #include "test/lizard_common_test.h" + #include "test/lizard_compress_test.h" + #else + #include "lizard_compress_liz.h" + #endif +#endif +#include "lizard_compress_lz4.h" +#include "entropy/huf.h" + + +/* ************************************* +* Local Macros +***************************************/ +#define DELTANEXT(p) chainTable[(p) & contentMask] +#define LIZARD_MINIMAL_HUFF_GAIN(comprSize) (comprSize + (comprSize/8) + 512) +#define LIZARD_MINIMAL_BLOCK_GAIN(comprSize) (comprSize + (comprSize/32) + 512) + + +/*-************************************ +* Local Utils +**************************************/ +int Lizard_versionNumber (void) { return LIZARD_VERSION_NUMBER; } +int Lizard_compressBound(int isize) { return LIZARD_COMPRESSBOUND(isize); } +int Lizard_sizeofState_MinLevel() { return Lizard_sizeofState(LIZARD_MIN_CLEVEL); } + + + +/* ************************************* +* Hash functions +***************************************/ +#define HASH_UPDATE_LIMIT 8 /* equal to MEM_read64 */ +static const U32 prime4bytes = 2654435761U; +static const U64 prime5bytes = 889523592379ULL; +static const U64 prime6bytes = 227718039650203ULL; +static const U64 prime7bytes = 58295818150454627ULL; + +#if MINMATCH == 3 +static const U32 prime3bytes = 506832829U; +static U32 Lizard_hash3(U32 u, U32 h) { return (u * prime3bytes) << (32-24) >> (32-h) ; } +static size_t Lizard_hash3Ptr(const void* ptr, U32 h) { return Lizard_hash3(MEM_read32(ptr), h); } +#endif + +static U32 Lizard_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } +static size_t Lizard_hash4Ptr(const void* ptr, U32 h) { return Lizard_hash4(MEM_read32(ptr), h); } + +static size_t Lizard_hash5(U64 u, U32 h) { return (size_t)((u * prime5bytes) << (64-40) >> (64-h)) ; } +static size_t Lizard_hash5Ptr(const void* p, U32 h) { return Lizard_hash5(MEM_read64(p), h); } + +static size_t Lizard_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; } +static size_t Lizard_hash6Ptr(const void* p, U32 h) { return Lizard_hash6(MEM_read64(p), h); } + +static size_t Lizard_hash7(U64 u, U32 h) { return (size_t)((u * prime7bytes) << (64-56) >> (64-h)) ; } +static size_t Lizard_hash7Ptr(const void* p, U32 h) { return Lizard_hash7(MEM_read64(p), h); } + +static size_t Lizard_hashPtr(const void* p, U32 hBits, U32 mls) +{ + switch(mls) + { + default: + case 4: return Lizard_hash4Ptr(p, hBits); + case 5: return Lizard_hash5Ptr(p, hBits); + case 6: return Lizard_hash6Ptr(p, hBits); + case 7: return Lizard_hash7Ptr(p, hBits); + } +} + + + + +/************************************** +* Internal functions +**************************************/ +/** Lizard_count_2segments() : +* can count match length with `ip` & `match` in 2 different segments. +* convention : on reaching mEnd, match count continue starting from iStart +*/ +static size_t Lizard_count_2segments(const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart) +{ + const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd); + size_t const matchLength = Lizard_count(ip, match, vEnd); + if (match + matchLength != mEnd) return matchLength; + return matchLength + Lizard_count(ip+matchLength, iStart, iEnd); +} + + +void Lizard_initBlock(Lizard_stream_t* ctx) +{ + ctx->offset16Ptr = ctx->offset16Base; + ctx->offset24Ptr = ctx->offset24Base; + ctx->lenPtr = ctx->lenBase; + ctx->literalsPtr = ctx->literalsBase; + ctx->flagsPtr = ctx->flagsBase; + ctx->last_off = LIZARD_INIT_LAST_OFFSET; /* reset last offset */ +} + + +FORCE_INLINE int Lizard_writeStream(int useHuff, Lizard_stream_t* ctx, BYTE* streamPtr, uint32_t streamLen, BYTE** op, BYTE* oend) +{ + if (useHuff && streamLen > 1024) { +#ifndef LIZARD_NO_HUFFMAN + int useHuffBuf; + if (*op + 6 > oend) { LIZARD_LOG_COMPRESS("*op[%p] + 6 > oend[%p]\n", *op, oend); return -1; } + + useHuffBuf = ((size_t)(oend - (*op + 6)) < HUF_compressBound(streamLen)); + if (useHuffBuf) { + if (streamLen > LIZARD_BLOCK_SIZE) { LIZARD_LOG_COMPRESS("streamLen[%d] > LIZARD_BLOCK_SIZE\n", streamLen); return -1; } + ctx->comprStreamLen = (U32)HUF_compress(ctx->huffBase, ctx->huffEnd - ctx->huffBase, streamPtr, streamLen); + } else { + ctx->comprStreamLen = (U32)HUF_compress(*op + 6, oend - (*op + 6), streamPtr, streamLen); + } + + if (!HUF_isError(ctx->comprStreamLen)) { + if (ctx->comprStreamLen > 0 && (LIZARD_MINIMAL_HUFF_GAIN(ctx->comprStreamLen) < streamLen)) { /* compressible */ + MEM_writeLE24(*op, streamLen); + MEM_writeLE24(*op+3, ctx->comprStreamLen); + if (useHuffBuf) { + if ((size_t)(oend - (*op + 6)) < ctx->comprStreamLen) { LIZARD_LOG_COMPRESS("*op[%p] oend[%p] comprStreamLen[%d]\n", *op, oend, (int)ctx->comprStreamLen); return -1; } + memcpy(*op + 6, ctx->huffBase, ctx->comprStreamLen); + } + *op += ctx->comprStreamLen + 6; + LIZARD_LOG_COMPRESS("HUF_compress streamLen=%d comprStreamLen=%d\n", (int)streamLen, (int)ctx->comprStreamLen); + return 1; + } else { LIZARD_LOG_COMPRESS("HUF_compress ERROR comprStreamLen=%d streamLen=%d\n", (int)ctx->comprStreamLen, (int)streamLen); } + } else { LIZARD_LOG_COMPRESS("HUF_compress ERROR %d: %s\n", (int)ctx->comprStreamLen, HUF_getErrorName(ctx->comprStreamLen)); } +#else + LIZARD_LOG_COMPRESS("compiled with LIZARD_NO_HUFFMAN\n"); + (void)ctx; + return -1; +#endif + } else ctx->comprStreamLen = 0; + + if (*op + 3 + streamLen > oend) { LIZARD_LOG_COMPRESS("*op[%p] + 3 + streamLen[%d] > oend[%p]\n", *op, streamLen, oend); return -1; } + MEM_writeLE24(*op, streamLen); + *op += 3; + memcpy(*op, streamPtr, streamLen); + *op += streamLen; + LIZARD_LOG_COMPRESS("Uncompressed streamLen=%d\n", (int)streamLen); + return 0; +} + + +int Lizard_writeBlock(Lizard_stream_t* ctx, const BYTE* ip, uint32_t inputSize, BYTE** op, BYTE* oend) +{ + int res; + uint32_t flagsLen = (uint32_t)(ctx->flagsPtr - ctx->flagsBase); + uint32_t literalsLen = (uint32_t)(ctx->literalsPtr - ctx->literalsBase); + uint32_t lenLen = (uint32_t)(ctx->lenPtr - ctx->lenBase); + uint32_t offset16Len = (uint32_t)(ctx->offset16Ptr - ctx->offset16Base); + uint32_t offset24Len = (uint32_t)(ctx->offset24Ptr - ctx->offset24Base); + uint32_t sum = flagsLen + literalsLen + lenLen + offset16Len + offset24Len; +#ifdef LIZARD_USE_LOGS + uint32_t comprFlagsLen, comprLiteralsLen; +#endif + + BYTE* start = *op; + + if ((literalsLen < WILDCOPYLENGTH) || (sum+5*3+1 > inputSize)) goto _write_uncompressed; + + *start = 0; + *op += 1; + + res = Lizard_writeStream(0, ctx, ctx->lenBase, lenLen, op, oend); + if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_LEN); + + res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_OFFSET16, ctx, ctx->offset16Base, offset16Len, op, oend); + if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_OFFSET16); + + res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_OFFSET24, ctx, ctx->offset24Base, offset24Len, op, oend); + if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_OFFSET24); + + res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_FLAGS, ctx, ctx->flagsBase, flagsLen, op, oend); + if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_FLAGS); +#ifdef LIZARD_USE_LOGS + comprFlagsLen = ctx->comprStreamLen; +#endif + + res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_LITERALS, ctx, ctx->literalsBase, literalsLen, op, oend); + if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_LITERALS); +#ifdef LIZARD_USE_LOGS + comprLiteralsLen = ctx->comprStreamLen; + sum = (int)(*op-start); +#endif + + if (LIZARD_MINIMAL_BLOCK_GAIN((uint32_t)(*op-start)) > inputSize) goto _write_uncompressed; + + LIZARD_LOG_COMPRESS("%d: total=%d block=%d flagsLen[%.2f%%]=%d comprFlagsLen[%.2f%%]=%d literalsLen[%.2f%%]=%d comprLiteralsLen[%.2f%%]=%d lenLen=%d offset16Len[%.2f%%]=%d offset24Len[%.2f%%]=%d\n", (int)(ip - ctx->srcBase), + (int)(*op - ctx->destBase), sum, (flagsLen*100.0)/sum, flagsLen, (comprFlagsLen*100.0)/sum, comprFlagsLen, (literalsLen*100.0)/sum, literalsLen, (comprLiteralsLen*100.0)/sum, comprLiteralsLen, + lenLen, (offset16Len*100.0)/sum, offset16Len, (offset24Len*100.0)/sum, offset24Len); + return 0; + +_write_uncompressed: + LIZARD_LOG_COMPRESS("%d: total=%d block=%d UNCOMPRESSED inputSize=%u outSize=%d\n", (int)(ip - ctx->srcBase), + (int)(*op - ctx->destBase), (int)(*op-start), inputSize, (int)(oend-start)); + if ((uint32_t)(oend - start) < inputSize + 4) goto _output_error; + *start = LIZARD_FLAG_UNCOMPRESSED; + *op = start + 1; + MEM_writeLE24(*op, inputSize); + *op += 3; + memcpy(*op, ip, inputSize); + *op += inputSize; + return 0; + +_output_error: + LIZARD_LOG_COMPRESS("Lizard_writeBlock ERROR size=%d/%d flagsLen=%d literalsLen=%d lenLen=%d offset16Len=%d offset24Len=%d\n", (int)(*op-start), (int)(oend-start), flagsLen, literalsLen, lenLen, offset16Len, offset24Len); + return 1; +} + + +FORCE_INLINE int Lizard_encodeSequence ( + Lizard_stream_t* ctx, + const BYTE** ip, + const BYTE** anchor, + size_t matchLength, + const BYTE* const match) +{ +#ifdef USE_LZ4_ONLY + return Lizard_encodeSequence_LZ4(ctx, ip, anchor, matchLength, match); +#else + if (ctx->params.decompressType == Lizard_coderwords_LZ4) + return Lizard_encodeSequence_LZ4(ctx, ip, anchor, matchLength, match); + + return Lizard_encodeSequence_LIZv1(ctx, ip, anchor, matchLength, match); +#endif +} + + +FORCE_INLINE int Lizard_encodeLastLiterals ( + Lizard_stream_t* ctx, + const BYTE** ip, + const BYTE** anchor) +{ + LIZARD_LOG_COMPRESS("Lizard_encodeLastLiterals Lizard_coderwords_LZ4=%d\n", ctx->params.decompressType == Lizard_coderwords_LZ4); +#ifdef USE_LZ4_ONLY + return Lizard_encodeLastLiterals_LZ4(ctx, ip, anchor); +#else + if (ctx->params.decompressType == Lizard_coderwords_LZ4) + return Lizard_encodeLastLiterals_LZ4(ctx, ip, anchor); + + return Lizard_encodeLastLiterals_LIZv1(ctx, ip, anchor); +#endif +} + + +/************************************** +* Include parsers +**************************************/ +#include "lizard_parser_hashchain.h" +#include "lizard_parser_nochain.h" +#include "lizard_parser_fast.h" +#include "lizard_parser_fastsmall.h" +#include "lizard_parser_fastbig.h" +#ifndef USE_LZ4_ONLY + #include "lizard_parser_optimal.h" + #include "lizard_parser_lowestprice.h" + #include "lizard_parser_pricefast.h" +#endif + + +int Lizard_verifyCompressionLevel(int compressionLevel) +{ + if (compressionLevel > LIZARD_MAX_CLEVEL) compressionLevel = LIZARD_MAX_CLEVEL; + if (compressionLevel < LIZARD_MIN_CLEVEL) compressionLevel = LIZARD_DEFAULT_CLEVEL; + return compressionLevel; +} + + +int Lizard_sizeofState(int compressionLevel) +{ + Lizard_parameters params; + U32 hashTableSize, chainTableSize; + + compressionLevel = Lizard_verifyCompressionLevel(compressionLevel); + params = Lizard_defaultParameters[compressionLevel - LIZARD_MIN_CLEVEL]; +// hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog3)+((size_t)1 << params.hashLog))); + hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog))); + chainTableSize = (U32)(sizeof(U32)*((size_t)1 << params.contentLog)); + + return sizeof(Lizard_stream_t) + hashTableSize + chainTableSize + LIZARD_COMPRESS_ADD_BUF + (int)LIZARD_COMPRESS_ADD_HUF; +} + + +static void Lizard_init(Lizard_stream_t* ctx, const BYTE* start) +{ + // No need to use memset() on tables as values are always bound checked +#ifdef LIZARD_RESET_MEM + MEM_INIT((void*)ctx->hashTable, 0, ctx->hashTableSize); + MEM_INIT(ctx->chainTable, 0x01, ctx->chainTableSize); +#endif + // printf("memset hashTable=%p hashEnd=%p chainTable=%p chainEnd=%p\n", ctx->hashTable, ((BYTE*)ctx->hashTable) + ctx->hashTableSize, ctx->chainTable, ((BYTE*)ctx->chainTable)+ctx->chainTableSize); + ctx->nextToUpdate = LIZARD_DICT_SIZE; + ctx->base = start - LIZARD_DICT_SIZE; + ctx->end = start; + ctx->dictBase = start - LIZARD_DICT_SIZE; + ctx->dictLimit = LIZARD_DICT_SIZE; + ctx->lowLimit = LIZARD_DICT_SIZE; + ctx->last_off = LIZARD_INIT_LAST_OFFSET; + ctx->litSum = 0; +} + + +/* if ctx==NULL memory is allocated and returned as value */ +Lizard_stream_t* Lizard_initStream(Lizard_stream_t* ctx, int compressionLevel) +{ + Lizard_parameters params; + U32 hashTableSize, chainTableSize; + void *tempPtr; + + compressionLevel = Lizard_verifyCompressionLevel(compressionLevel); + params = Lizard_defaultParameters[compressionLevel - LIZARD_MIN_CLEVEL]; +// hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog3)+((size_t)1 << params.hashLog))); + hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog))); + chainTableSize = (U32)(sizeof(U32)*((size_t)1 << params.contentLog)); + + if (!ctx) + { + ctx = (Lizard_stream_t*)malloc(sizeof(Lizard_stream_t) + hashTableSize + chainTableSize + LIZARD_COMPRESS_ADD_BUF + LIZARD_COMPRESS_ADD_HUF); + if (!ctx) { printf("ERROR: Cannot allocate %d MB (compressionLevel=%d)\n", (int)(sizeof(Lizard_stream_t) + hashTableSize + chainTableSize)>>20, compressionLevel); return 0; } + LIZARD_LOG_COMPRESS("Allocated %d MB (compressionLevel=%d)\n", (int)(sizeof(Lizard_stream_t) + hashTableSize + chainTableSize)>>20, compressionLevel); + ctx->allocatedMemory = sizeof(Lizard_stream_t) + hashTableSize + chainTableSize + LIZARD_COMPRESS_ADD_BUF + (U32)LIZARD_COMPRESS_ADD_HUF; + // printf("malloc from=%p to=%p hashTable=%p hashEnd=%p chainTable=%p chainEnd=%p\n", ctx, ((BYTE*)ctx)+sizeof(Lizard_stream_t) + hashTableSize + chainTableSize, ctx->hashTable, ((BYTE*)ctx->hashTable) + hashTableSize, ctx->chainTable, ((BYTE*)ctx->chainTable)+chainTableSize); + } + + tempPtr = ctx; + ctx->hashTable = (U32*)(tempPtr) + sizeof(Lizard_stream_t)/4; + ctx->hashTableSize = hashTableSize; + ctx->chainTable = ctx->hashTable + hashTableSize/4; + ctx->chainTableSize = chainTableSize; + ctx->params = params; + ctx->compressionLevel = (unsigned)compressionLevel; + if (compressionLevel < 30) + ctx->huffType = 0; + else + ctx->huffType = LIZARD_FLAG_LITERALS + LIZARD_FLAG_FLAGS; // + LIZARD_FLAG_OFFSET16 + LIZARD_FLAG_OFFSET24; + + ctx->literalsBase = (BYTE*)ctx->hashTable + ctx->hashTableSize + ctx->chainTableSize; + ctx->flagsBase = ctx->literalsEnd = ctx->literalsBase + LIZARD_BLOCK_SIZE_PAD; + ctx->lenBase = ctx->flagsEnd = ctx->flagsBase + LIZARD_BLOCK_SIZE_PAD; + ctx->offset16Base = ctx->lenEnd = ctx->lenBase + LIZARD_BLOCK_SIZE_PAD; + ctx->offset24Base = ctx->offset16End = ctx->offset16Base + LIZARD_BLOCK_SIZE_PAD; + ctx->huffBase = ctx->offset24End = ctx->offset24Base + LIZARD_BLOCK_SIZE_PAD; + ctx->huffEnd = ctx->huffBase + LIZARD_COMPRESS_ADD_HUF; + + return ctx; +} + + + +Lizard_stream_t* Lizard_createStream(int compressionLevel) +{ + Lizard_stream_t* ctx = Lizard_initStream(NULL, compressionLevel); + if (ctx) ctx->base = NULL; + return ctx; +} + + +/* initialization */ +Lizard_stream_t* Lizard_resetStream(Lizard_stream_t* ctx, int compressionLevel) +{ + size_t wanted = Lizard_sizeofState(compressionLevel); + + if (ctx->allocatedMemory < wanted) { + Lizard_freeStream(ctx); + ctx = Lizard_createStream(compressionLevel); + } else { + Lizard_initStream(ctx, compressionLevel); + } + + if (ctx) ctx->base = NULL; + return ctx; +} + + +int Lizard_freeStream(Lizard_stream_t* ctx) +{ + if (ctx) { + free(ctx); + } + return 0; +} + + +int Lizard_loadDict(Lizard_stream_t* Lizard_streamPtr, const char* dictionary, int dictSize) +{ + Lizard_stream_t* ctxPtr = (Lizard_stream_t*) Lizard_streamPtr; + if (dictSize > LIZARD_DICT_SIZE) { + dictionary += dictSize - LIZARD_DICT_SIZE; + dictSize = LIZARD_DICT_SIZE; + } + Lizard_init(ctxPtr, (const BYTE*)dictionary); + if (dictSize >= HASH_UPDATE_LIMIT) Lizard_Insert (ctxPtr, (const BYTE*)dictionary + (dictSize - (HASH_UPDATE_LIMIT-1))); + ctxPtr->end = (const BYTE*)dictionary + dictSize; + return dictSize; +} + + +static void Lizard_setExternalDict(Lizard_stream_t* ctxPtr, const BYTE* newBlock) +{ + if (ctxPtr->end >= ctxPtr->base + HASH_UPDATE_LIMIT) Lizard_Insert (ctxPtr, ctxPtr->end - (HASH_UPDATE_LIMIT-1)); /* Referencing remaining dictionary content */ + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); + ctxPtr->dictBase = ctxPtr->base; + ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ +} + + +/* dictionary saving */ +int Lizard_saveDict (Lizard_stream_t* Lizard_streamPtr, char* safeBuffer, int dictSize) +{ + Lizard_stream_t* const ctx = (Lizard_stream_t*)Lizard_streamPtr; + int const prefixSize = (int)(ctx->end - (ctx->base + ctx->dictLimit)); + if (dictSize > LIZARD_DICT_SIZE) dictSize = LIZARD_DICT_SIZE; + if (dictSize < 4) dictSize = 0; + if (dictSize > prefixSize) dictSize = prefixSize; + memmove(safeBuffer, ctx->end - dictSize, dictSize); + { U32 const endIndex = (U32)(ctx->end - ctx->base); + ctx->end = (const BYTE*)safeBuffer + dictSize; + ctx->base = ctx->end - endIndex; + ctx->dictLimit = endIndex - dictSize; + ctx->lowLimit = endIndex - dictSize; + if (ctx->nextToUpdate < ctx->dictLimit) ctx->nextToUpdate = ctx->dictLimit; + } + return dictSize; +} + +FORCE_INLINE int Lizard_compress_generic ( + void* ctxvoid, + const char* source, + char* dest, + int inputSize, + int maxOutputSize) +{ + Lizard_stream_t* ctx = (Lizard_stream_t*) ctxvoid; + size_t dictSize = (size_t)(ctx->end - ctx->base) - ctx->dictLimit; + const BYTE* ip = (const BYTE*) source; + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + maxOutputSize; + int res; + + (void)dictSize; + LIZARD_LOG_COMPRESS("Lizard_compress_generic source=%p inputSize=%d dest=%p maxOutputSize=%d cLevel=%d dictBase=%p dictSize=%d\n", source, inputSize, dest, maxOutputSize, ctx->compressionLevel, ctx->dictBase, (int)dictSize); + *op++ = (BYTE)ctx->compressionLevel; + maxOutputSize--; // can be lower than 0 + ctx->end += inputSize; + ctx->srcBase = ctx->off24pos = ip; + ctx->destBase = (BYTE*)dest; + + while (inputSize > 0) + { + int inputPart = MIN(LIZARD_BLOCK_SIZE, inputSize); + + if (ctx->huffType) Lizard_rescaleFreqs(ctx); + Lizard_initBlock(ctx); + ctx->diffBase = ip; + + switch(ctx->params.parserType) + { + default: + case Lizard_parser_fastSmall: + res = Lizard_compress_fastSmall(ctx, ip, ip+inputPart); break; + case Lizard_parser_fast: + res = Lizard_compress_fast(ctx, ip, ip+inputPart); break; + case Lizard_parser_noChain: + res = Lizard_compress_noChain(ctx, ip, ip+inputPart); break; + case Lizard_parser_hashChain: + res = Lizard_compress_hashChain(ctx, ip, ip+inputPart); break; +#ifndef USE_LZ4_ONLY + case Lizard_parser_fastBig: + res = Lizard_compress_fastBig(ctx, ip, ip+inputPart); break; + case Lizard_parser_priceFast: + res = Lizard_compress_priceFast(ctx, ip, ip+inputPart); break; + case Lizard_parser_lowestPrice: + res = Lizard_compress_lowestPrice(ctx, ip, ip+inputPart); break; + case Lizard_parser_optimalPrice: + case Lizard_parser_optimalPriceBT: + res = Lizard_compress_optimalPrice(ctx, ip, ip+inputPart); break; +#else + case Lizard_parser_priceFast: + case Lizard_parser_lowestPrice: + case Lizard_parser_optimalPrice: + case Lizard_parser_optimalPriceBT: + res = 0; +#endif + } + + LIZARD_LOG_COMPRESS("Lizard_compress_generic res=%d inputPart=%d \n", res, inputPart); + if (res <= 0) return res; + + if (Lizard_writeBlock(ctx, ip, inputPart, &op, oend)) goto _output_error; + + ip += inputPart; + inputSize -= inputPart; + LIZARD_LOG_COMPRESS("Lizard_compress_generic in=%d out=%d\n", (int)(ip-(const BYTE*)source), (int)(op-(BYTE*)dest)); + } + + LIZARD_LOG_COMPRESS("Lizard_compress_generic total=%d\n", (int)(op-(BYTE*)dest)); + return (int)(op-(BYTE*)dest); +_output_error: + LIZARD_LOG_COMPRESS("Lizard_compress_generic ERROR\n"); + return 0; +} + + +int Lizard_compress_continue (Lizard_stream_t* ctxPtr, + const char* source, char* dest, + int inputSize, int maxOutputSize) +{ + /* auto-init if required */ + if (ctxPtr->base == NULL) Lizard_init(ctxPtr, (const BYTE*) source); + + /* Check overflow */ + if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { + size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; + if (dictSize > LIZARD_DICT_SIZE) dictSize = LIZARD_DICT_SIZE; + Lizard_loadDict((Lizard_stream_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); + } + + /* Check if blocks follow each other */ + if ((const BYTE*)source != ctxPtr->end) + Lizard_setExternalDict(ctxPtr, (const BYTE*)source); + + /* Check overlapping input/dictionary space */ + { const BYTE* sourceEnd = (const BYTE*) source + inputSize; + const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; + const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; + if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) { + if (sourceEnd > dictEnd) sourceEnd = dictEnd; + ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); + if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; + } + } + + return Lizard_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize); +} + + +int Lizard_compress_extState (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) +{ + Lizard_stream_t* ctx = (Lizard_stream_t*) state; + if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ + + /* initialize stream */ + Lizard_initStream(ctx, compressionLevel); + Lizard_init((Lizard_stream_t*)state, (const BYTE*)src); + + return Lizard_compress_generic (state, src, dst, srcSize, maxDstSize); +} + + +int Lizard_compress(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) +{ + int cSize; + Lizard_stream_t* statePtr = Lizard_createStream(compressionLevel); + + if (!statePtr) return 0; + cSize = Lizard_compress_extState(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); + + Lizard_freeStream(statePtr); + return cSize; +} + + +/************************************** +* Level1 functions +**************************************/ +int Lizard_compress_extState_MinLevel(void* state, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return Lizard_compress_extState(state, source, dest, inputSize, maxOutputSize, LIZARD_MIN_CLEVEL); +} + +int Lizard_compress_MinLevel(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return Lizard_compress(source, dest, inputSize, maxOutputSize, LIZARD_MIN_CLEVEL); +} + +Lizard_stream_t* Lizard_createStream_MinLevel(void) +{ + return Lizard_createStream(LIZARD_MIN_CLEVEL); +} + +Lizard_stream_t* Lizard_resetStream_MinLevel(Lizard_stream_t* Lizard_stream) +{ + return Lizard_resetStream (Lizard_stream, LIZARD_MIN_CLEVEL); +} diff --git a/contrib/lizard/lib/lizard_compress.h b/contrib/lizard/lib/lizard_compress.h new file mode 100644 index 00000000000..d84246e2099 --- /dev/null +++ b/contrib/lizard/lib/lizard_compress.h @@ -0,0 +1,208 @@ +/* + Lizard - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2016, Yann Collet + Copyright (C) 2016-2017, Przemyslaw Skibinski + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ +#ifndef LIZARD_H_2983 +#define LIZARD_H_2983 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* + * lizard_compress.h provides block compression functions. It gives full buffer control to user. + * Block compression functions are not-enough to send information, + * since it's still necessary to provide metadata (such as compressed size), + * and each application can do it in whichever way it wants. + * For interoperability, there is Lizard frame specification (lizard_Frame_format.md). + * A library is provided to take care of it, see lizard_frame.h. +*/ + + +/*^*************************************************************** +* Export parameters +*****************************************************************/ +/* +* LIZARD_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(LIZARD_DLL_EXPORT) && (LIZARD_DLL_EXPORT==1) +# define LIZARDLIB_API __declspec(dllexport) +#elif defined(LIZARD_DLL_IMPORT) && (LIZARD_DLL_IMPORT==1) +# define LIZARDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define LIZARDLIB_API +#endif + + +/*-************************************ +* Version +**************************************/ +#define LIZARD_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LIZARD_VERSION_MINOR 0 /* for new (non-breaking) interface capabilities */ +#define LIZARD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ + +#define LIZARD_VERSION_NUMBER (LIZARD_VERSION_MAJOR *100*100 + LIZARD_VERSION_MINOR *100 + LIZARD_VERSION_RELEASE) +int Lizard_versionNumber (void); + +#define LIZARD_LIB_VERSION LIZARD_VERSION_MAJOR.LIZARD_VERSION_MINOR.LIZARD_VERSION_RELEASE +#define LIZARD_QUOTE(str) #str +#define LIZARD_EXPAND_AND_QUOTE(str) LIZARD_QUOTE(str) +#define LIZARD_VERSION_STRING LIZARD_EXPAND_AND_QUOTE(LIZARD_LIB_VERSION) +const char* Lizard_versionString (void); + +typedef struct Lizard_stream_s Lizard_stream_t; + +#define LIZARD_MIN_CLEVEL 10 /* minimum compression level */ +#ifndef LIZARD_NO_HUFFMAN + #define LIZARD_MAX_CLEVEL 49 /* maximum compression level */ +#else + #define LIZARD_MAX_CLEVEL 29 /* maximum compression level */ +#endif +#define LIZARD_DEFAULT_CLEVEL 17 + + +/*-************************************ +* Simple Functions +**************************************/ + +LIZARDLIB_API int Lizard_compress (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); + +/* +Lizard_compress() : + Compresses 'sourceSize' bytes from buffer 'source' + into already allocated 'dest' buffer of size 'maxDestSize'. + Compression is guaranteed to succeed if 'maxDestSize' >= Lizard_compressBound(sourceSize). + It also runs faster, so it's a recommended setting. + If the function cannot compress 'source' into a more limited 'dest' budget, + compression stops *immediately*, and the function result is zero. + As a consequence, 'dest' content is not valid. + This function never writes outside 'dest' buffer, nor read outside 'source' buffer. + sourceSize : Max supported value is LIZARD_MAX_INPUT_VALUE + maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) + return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) + or 0 if compression fails +*/ + + +/*-************************************ +* Advanced Functions +**************************************/ +#define LIZARD_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LIZARD_BLOCK_SIZE (1<<17) +#define LIZARD_BLOCK64K_SIZE (1<<16) +#define LIZARD_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LIZARD_MAX_INPUT_SIZE ? 0 : (isize) + 1 + 1 + ((isize/LIZARD_BLOCK_SIZE)+1)*4) + + +/*! +Lizard_compressBound() : + Provides the maximum size that Lizard compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LIZARD_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that Lizard_compress() compress faster when dest buffer size is >= Lizard_compressBound(srcSize) + inputSize : max supported value is LIZARD_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is too large ( > LIZARD_MAX_INPUT_SIZE) +*/ +LIZARDLIB_API int Lizard_compressBound(int inputSize); + + +/*! +Lizard_compress_extState() : + Same compression function, just using an externally allocated memory space to store compression state. + Use Lizard_sizeofState() to know how much memory must be allocated, + and allocate it on 8-bytes boundaries (using malloc() typically). + Then, provide it as 'void* state' to compression function. +*/ +LIZARDLIB_API int Lizard_sizeofState(int compressionLevel); + +LIZARDLIB_API int Lizard_compress_extState(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); + + + +/*-********************************************* +* Streaming Compression Functions +***********************************************/ + +/*! Lizard_createStream() will allocate and initialize an `Lizard_stream_t` structure. + * Lizard_freeStream() releases its memory. + * In the context of a DLL (liblizard), please use these methods rather than the static struct. + * They are more future proof, in case of a change of `Lizard_stream_t` size. + */ +LIZARDLIB_API Lizard_stream_t* Lizard_createStream(int compressionLevel); +LIZARDLIB_API int Lizard_freeStream (Lizard_stream_t* streamPtr); + + +/*! Lizard_resetStream() : + * Use this function to reset/reuse an allocated `Lizard_stream_t` structure + */ +LIZARDLIB_API Lizard_stream_t* Lizard_resetStream (Lizard_stream_t* streamPtr, int compressionLevel); + + +/*! Lizard_loadDict() : + * Use this function to load a static dictionary into Lizard_stream. + * Any previous data will be forgotten, only 'dictionary' will remain in memory. + * Loading a size of 0 is allowed. + * Return : dictionary size, in bytes (necessarily <= LIZARD_DICT_SIZE) + */ +LIZARDLIB_API int Lizard_loadDict (Lizard_stream_t* streamPtr, const char* dictionary, int dictSize); + + +/*! Lizard_compress_continue() : + * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. + * Important : Previous data blocks are assumed to still be present and unmodified ! + * 'dst' buffer must be already allocated. + * If maxDstSize >= Lizard_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. + */ +LIZARDLIB_API int Lizard_compress_continue (Lizard_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize); + + +/*! Lizard_saveDict() : + * If previously compressed data block is not guaranteed to remain available at its memory location, + * save it into a safer place (char* safeBuffer). + * Note : you don't need to call Lizard_loadDict() afterwards, + * dictionary is immediately usable, you can therefore call Lizard_compress_continue(). + * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + */ +LIZARDLIB_API int Lizard_saveDict (Lizard_stream_t* streamPtr, char* safeBuffer, int dictSize); + + + + + +#if defined (__cplusplus) +} +#endif + +#endif /* LIZARD_H_2983827168210 */ diff --git a/contrib/lizard/lib/lizard_compress_liz.h b/contrib/lizard/lib/lizard_compress_liz.h new file mode 100644 index 00000000000..f531f36703a --- /dev/null +++ b/contrib/lizard/lib/lizard_compress_liz.h @@ -0,0 +1,301 @@ +#define LIZARD_FREQ_DIV 5 + +FORCE_INLINE void Lizard_setLog2Prices(Lizard_stream_t* ctx) +{ + ctx->log2LitSum = Lizard_highbit32(ctx->litSum+1); + ctx->log2FlagSum = Lizard_highbit32(ctx->flagSum+1); +} + + +MEM_STATIC void Lizard_rescaleFreqs(Lizard_stream_t* ctx) +{ + unsigned u; + + ctx->cachedLiterals = NULL; + ctx->cachedPrice = ctx->cachedLitLength = 0; + + ctx->litPriceSum = 0; + + if (ctx->litSum == 0) { + ctx->litSum = 2 * 256; + ctx->flagSum = 2 * 256; + + for (u=0; u < 256; u++) { + ctx->litFreq[u] = 2; + ctx->flagFreq[u] = 2; + } + } else { + ctx->litSum = 0; + ctx->flagSum = 0; + + for (u=0; u < 256; u++) { + ctx->litFreq[u] = 1 + (ctx->litFreq[u]>>LIZARD_FREQ_DIV); + ctx->litSum += ctx->litFreq[u]; + ctx->flagFreq[u] = 1 + (ctx->flagFreq[u]>>LIZARD_FREQ_DIV); + ctx->flagSum += ctx->flagFreq[u]; + } + } + + Lizard_setLog2Prices(ctx); +} + + +FORCE_INLINE int Lizard_encodeSequence_LIZv1 ( + Lizard_stream_t* ctx, + const BYTE** ip, + const BYTE** anchor, + size_t matchLength, + const BYTE* const match) +{ + U32 offset = (U32)(*ip - match); + size_t length = (size_t)(*ip - *anchor); + BYTE* token = (ctx->flagsPtr)++; + + if (length > 0 || offset < LIZARD_MAX_16BIT_OFFSET) { + /* Encode Literal length */ + // if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - length - LIZARD_LENGTH_SIZE_LIZv1(length) - WILDCOPYLENGTH)) { LIZARD_LOG_COMPRESS_LIZv1("encodeSequence overflow1\n"); return 1; } /* Check output limit */ + if (length >= MAX_SHORT_LITLEN) + { size_t len; + *token = MAX_SHORT_LITLEN; + len = length - MAX_SHORT_LITLEN; + if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; } + else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; } + else *(ctx->literalsPtr)++ = (BYTE)len; + } + else *token = (BYTE)length; + + /* Copy Literals */ + Lizard_wildCopy(ctx->literalsPtr, *anchor, (ctx->literalsPtr) + length); +#ifndef LIZARD_NO_HUFFMAN + if (ctx->huffType) { + ctx->litSum += (U32)length; + ctx->litPriceSum += (U32)(length * ctx->log2LitSum); + { U32 u; + for (u=0; u < length; u++) { + ctx->litPriceSum -= Lizard_highbit32(ctx->litFreq[ctx->literalsPtr[u]]+1); + ctx->litFreq[ctx->literalsPtr[u]]++; + } } + } +#endif + ctx->literalsPtr += length; + + + if (offset >= LIZARD_MAX_16BIT_OFFSET) { + COMPLOG_CODEWORDS_LIZv1("T32+ literal=%u match=%u offset=%d\n", (U32)length, 0, 0); + *token+=(1<huffType) { + ctx->flagFreq[*token]++; + ctx->flagSum++; + } +#endif + token = (ctx->flagsPtr)++; + } + } + + /* Encode Offset */ + if (offset >= LIZARD_MAX_16BIT_OFFSET) // 24-bit offset + { + if (matchLength < MM_LONGOFF) printf("ERROR matchLength=%d/%d\n", (int)matchLength, MM_LONGOFF), exit(1); + + // if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - 8 /*LIZARD_LENGTH_SIZE_LIZv1(length)*/)) { LIZARD_LOG_COMPRESS_LIZv1("encodeSequence overflow2\n"); return 1; } /* Check output limit */ + if (matchLength - MM_LONGOFF >= LIZARD_LAST_LONG_OFF) + { + size_t len = matchLength - MM_LONGOFF - LIZARD_LAST_LONG_OFF; + *token = LIZARD_LAST_LONG_OFF; + if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; } + else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; } + else *(ctx->literalsPtr)++ = (BYTE)len; + COMPLOG_CODEWORDS_LIZv1("T31 literal=%u match=%u offset=%d\n", 0, (U32)matchLength, offset); + } + else + { + COMPLOG_CODEWORDS_LIZv1("T0-30 literal=%u match=%u offset=%d\n", 0, (U32)matchLength, offset); + *token = (BYTE)(matchLength - MM_LONGOFF); + } + + MEM_writeLE24(ctx->offset24Ptr, offset); + ctx->offset24Ptr += 3; + ctx->last_off = offset; + ctx->off24pos = *ip; + } + else + { + COMPLOG_CODEWORDS_LIZv1("T32+ literal=%u match=%u offset=%d\n", (U32)length, (U32)matchLength, offset); + if (offset == 0) + { + *token+=(1<last_off); exit(1); } + + ctx->last_off = offset; + MEM_writeLE16(ctx->offset16Ptr, (U16)ctx->last_off); ctx->offset16Ptr += 2; + } + + /* Encode MatchLength */ + length = matchLength; + // if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - 5 /*LIZARD_LENGTH_SIZE_LIZv1(length)*/)) { LIZARD_LOG_COMPRESS_LIZv1("encodeSequence overflow2\n"); return 1; } /* Check output limit */ + if (length >= MAX_SHORT_MATCHLEN) { + *token += (BYTE)(MAX_SHORT_MATCHLEN<= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(length)); ctx->literalsPtr += 4; } + else if (length >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(length)); ctx->literalsPtr += 3; } + else *(ctx->literalsPtr)++ = (BYTE)length; + } + else *token += (BYTE)(length<huffType) { + ctx->flagFreq[*token]++; + ctx->flagSum++; + Lizard_setLog2Prices(ctx); + } +#endif + + /* Prepare next loop */ + *ip += matchLength; + *anchor = *ip; + + return 0; +} + + +FORCE_INLINE int Lizard_encodeLastLiterals_LIZv1 ( + Lizard_stream_t* ctx, + const BYTE** ip, + const BYTE** anchor) +{ + size_t length = (int)(*ip - *anchor); + (void)ctx; + + memcpy(ctx->literalsPtr, *anchor, length); + ctx->literalsPtr += length; + return 0; +} + + +#define LIZARD_PRICE_MULT 1 +#define LIZARD_GET_TOKEN_PRICE_LIZv1(token) (LIZARD_PRICE_MULT * (ctx->log2FlagSum - Lizard_highbit32(ctx->flagFreq[token]+1))) + + +FORCE_INLINE size_t Lizard_get_price_LIZv1(Lizard_stream_t* const ctx, int rep, const BYTE *ip, const BYTE *off24pos, size_t litLength, U32 offset, size_t matchLength) +{ + size_t price = 0; + BYTE token = 0; +#ifndef LIZARD_NO_HUFFMAN + const BYTE* literals = ip - litLength; + U32 u; + if ((ctx->huffType) && (ctx->params.parserType != Lizard_parser_lowestPrice)) { + if (ctx->cachedLiterals == literals && litLength >= ctx->cachedLitLength) { + size_t const additional = litLength - ctx->cachedLitLength; + const BYTE* literals2 = ctx->cachedLiterals + ctx->cachedLitLength; + price = ctx->cachedPrice + LIZARD_PRICE_MULT * additional * ctx->log2LitSum; + for (u=0; u < additional; u++) + price -= LIZARD_PRICE_MULT * Lizard_highbit32(ctx->litFreq[literals2[u]]+1); + ctx->cachedPrice = (U32)price; + ctx->cachedLitLength = (U32)litLength; + } else { + price = LIZARD_PRICE_MULT * litLength * ctx->log2LitSum; + for (u=0; u < litLength; u++) + price -= LIZARD_PRICE_MULT * Lizard_highbit32(ctx->litFreq[literals[u]]+1); + + if (litLength >= 12) { + ctx->cachedLiterals = literals; + ctx->cachedPrice = (U32)price; + ctx->cachedLitLength = (U32)litLength; + } + } + } + else + price += 8*litLength; /* Copy Literals */ +#else + price += 8*litLength; /* Copy Literals */ + (void)ip; + (void)ctx; +#endif + + (void)off24pos; + (void)rep; + + if (litLength > 0 || offset < LIZARD_MAX_16BIT_OFFSET) { + /* Encode Literal length */ + if (litLength >= MAX_SHORT_LITLEN) + { size_t len = litLength - MAX_SHORT_LITLEN; + token = MAX_SHORT_LITLEN; + if (len >= (1<<16)) price += 32; + else if (len >= 254) price += 24; + else price += 8; + } + else token = (BYTE)litLength; + + if (offset >= LIZARD_MAX_16BIT_OFFSET) { + token+=(1<huffType && ctx->params.parserType != Lizard_parser_lowestPrice) + price += LIZARD_GET_TOKEN_PRICE_LIZv1(token); + else + price += 8; + } + } + + /* Encode Offset */ + if (offset >= LIZARD_MAX_16BIT_OFFSET) { // 24-bit offset + if (matchLength < MM_LONGOFF) return LIZARD_MAX_PRICE; // error + + if (matchLength - MM_LONGOFF >= LIZARD_LAST_LONG_OFF) { + size_t len = matchLength - MM_LONGOFF - LIZARD_LAST_LONG_OFF; + token = LIZARD_LAST_LONG_OFF; + if (len >= (1<<16)) price += 32; + else if (len >= 254) price += 24; + else price += 8; + } else { + token = (BYTE)(matchLength - MM_LONGOFF); + } + + price += 24; + } else { + size_t length; + if (offset == 0) { + token+=(1<= MAX_SHORT_MATCHLEN) { + token += (BYTE)(MAX_SHORT_MATCHLEN<= (1<<16)) price += 32; + else if (length >= 254) price += 24; + else price += 8; + } + else token += (BYTE)(length< 0 || matchLength > 0) { + int offset_load = Lizard_highbit32(offset); + if (ctx->huffType) { + price += ((offset_load>=20) ? ((offset_load-19)*4) : 0); + price += 4 + (matchLength==1); + } else { + price += ((offset_load>=16) ? ((offset_load-15)*4) : 0); + price += 6 + (matchLength==1); + } + if (ctx->huffType && ctx->params.parserType != Lizard_parser_lowestPrice) + price += LIZARD_GET_TOKEN_PRICE_LIZv1(token); + else + price += 8; + } else { + if (ctx->huffType && ctx->params.parserType != Lizard_parser_lowestPrice) + price += LIZARD_GET_TOKEN_PRICE_LIZv1(token); // 1=better ratio + } + + return price; +} diff --git a/contrib/lizard/lib/lizard_compress_lz4.h b/contrib/lizard/lib/lizard_compress_lz4.h new file mode 100644 index 00000000000..00686b6837e --- /dev/null +++ b/contrib/lizard/lib/lizard_compress_lz4.h @@ -0,0 +1,162 @@ +#define LIZARD_LENGTH_SIZE_LZ4(len) ((len >= (1<<16)+RUN_MASK_LZ4) ? 5 : ((len >= 254+RUN_MASK_LZ4) ? 3 : ((len >= RUN_MASK_LZ4) ? 1 : 0))) + +FORCE_INLINE int Lizard_encodeSequence_LZ4 ( + Lizard_stream_t* ctx, + const BYTE** ip, + const BYTE** anchor, + size_t matchLength, + const BYTE* const match) +{ + size_t length = (size_t)(*ip - *anchor); + BYTE* token = (ctx->flagsPtr)++; + (void) ctx; + + COMPLOG_CODEWORDS_LZ4("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); + + /* Encode Literal length */ + // if (ctx->literalsPtr > ctx->literalsEnd - length - LIZARD_LENGTH_SIZE_LZ4(length) - 2 - WILDCOPYLENGTH) { LIZARD_LOG_COMPRESS_LZ4("encodeSequence overflow1\n"); return 1; } /* Check output limit */ + if (length >= RUN_MASK_LZ4) + { size_t len = length - RUN_MASK_LZ4; + *token = RUN_MASK_LZ4; + if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; } + else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; } + else *(ctx->literalsPtr)++ = (BYTE)len; + } + else *token = (BYTE)length; + + /* Copy Literals */ + if (length > 0) { + Lizard_wildCopy(ctx->literalsPtr, *anchor, (ctx->literalsPtr) + length); +#if 0 //def LIZARD_USE_HUFFMAN + ctx->litSum += (U32)length; + ctx->litPriceSum += (U32)(length * ctx->log2LitSum); + { U32 u; + for (u=0; u < length; u++) { + ctx->litPriceSum -= Lizard_highbit32(ctx->litFreq[ctx->literalsPtr[u]]+1); + ctx->litFreq[ctx->literalsPtr[u]]++; + } } +#endif + ctx->literalsPtr += length; + } + + /* Encode Offset */ + MEM_writeLE16(ctx->literalsPtr, (U16)(*ip-match)); + ctx->literalsPtr+=2; + + /* Encode MatchLength */ + length = matchLength - MINMATCH; + // if (ctx->literalsPtr > ctx->literalsEnd - 5 /*LIZARD_LENGTH_SIZE_LZ4(length)*/) { LIZARD_LOG_COMPRESS_LZ4("encodeSequence overflow2\n"); return 1; } /* Check output limit */ + if (length >= ML_MASK_LZ4) { + *token += (BYTE)(ML_MASK_LZ4<= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(length)); ctx->literalsPtr += 4; } + else if (length >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(length)); ctx->literalsPtr += 3; } + else *(ctx->literalsPtr)++ = (BYTE)length; + } + else *token += (BYTE)(length<huffType) { + ctx->flagFreq[*token]++; + ctx->flagSum++; + Lizard_setLog2Prices(ctx); + } +#endif + + /* Prepare next loop */ + *ip += matchLength; + *anchor = *ip; + + return 0; +} + + +FORCE_INLINE int Lizard_encodeLastLiterals_LZ4 ( + Lizard_stream_t* ctx, + const BYTE** ip, + const BYTE** anchor) +{ + size_t length = (int)(*ip - *anchor); + + (void)ctx; + + memcpy(ctx->literalsPtr, *anchor, length); + ctx->literalsPtr += length; + return 0; +} + + +#define LIZARD_GET_TOKEN_PRICE_LZ4(token) (ctx->log2FlagSum - Lizard_highbit32(ctx->flagFreq[token]+1)) + +FORCE_INLINE size_t Lizard_get_price_LZ4(Lizard_stream_t* const ctx, const BYTE *ip, const size_t litLength, U32 offset, size_t matchLength) +{ + size_t price = 0; + BYTE token = 0; +#if 0 //def LIZARD_USE_HUFFMAN + const BYTE* literals = ip - litLength; + U32 u; + + if (ctx->cachedLiterals == literals && litLength >= ctx->cachedLitLength) { + size_t const additional = litLength - ctx->cachedLitLength; + const BYTE* literals2 = ctx->cachedLiterals + ctx->cachedLitLength; + price = ctx->cachedPrice + additional * ctx->log2LitSum; + for (u=0; u < additional; u++) + price -= Lizard_highbit32(ctx->litFreq[literals2[u]]+1); + ctx->cachedPrice = (U32)price; + ctx->cachedLitLength = (U32)litLength; + } else { + price = litLength * ctx->log2LitSum; + for (u=0; u < litLength; u++) + price -= Lizard_highbit32(ctx->litFreq[literals[u]]+1); + + if (litLength >= 12) { + ctx->cachedLiterals = literals; + ctx->cachedPrice = (U32)price; + ctx->cachedLitLength = (U32)litLength; + } + } +#else + price += 8*litLength; /* Copy Literals */ + (void)ip; + (void)ctx; +#endif + + /* Encode Literal length */ + if (litLength >= RUN_MASK_LZ4) { + size_t len = litLength - RUN_MASK_LZ4; + token = RUN_MASK_LZ4; + if (len >= (1<<16)) price += 32; + else if (len >= 254) price += 24; + else price += 8; + } + else token = (BYTE)litLength; + + + /* Encode MatchLength */ + if (offset) { + size_t length; + price += 16; /* Encode Offset */ + + if (offset < 8) return LIZARD_MAX_PRICE; // error + if (matchLength < MINMATCH) return LIZARD_MAX_PRICE; // error + + length = matchLength - MINMATCH; + if (length >= ML_MASK_LZ4) { + token += (BYTE)(ML_MASK_LZ4<= (1<<16)) price += 32; + else if (length >= 254) price += 24; + else price += 8; + } + else token += (BYTE)(length<huffType) { + if (offset > 0 || matchLength > 0) price += 2; + price += LIZARD_GET_TOKEN_PRICE_LZ4(token); + } else { + price += 8; // token + } + + return price; +} diff --git a/contrib/lizard/lib/lizard_decompress.c b/contrib/lizard/lib/lizard_decompress.c new file mode 100644 index 00000000000..df2eb699deb --- /dev/null +++ b/contrib/lizard/lib/lizard_decompress.c @@ -0,0 +1,372 @@ +/* + Lizard - Fast LZ compression algorithm + Copyright (C) 2011-2016, Yann Collet + Copyright (C) 2016-2017, Przemyslaw Skibinski + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + + +/************************************** +* Includes +**************************************/ +//#define LIZARD_STATS 1 // 0=simple stats, 1=more, 2=full +#ifdef LIZARD_STATS + #include "test/lizard_stats.h" +#endif +#include "lizard_compress.h" +#include "lizard_decompress.h" +#include "lizard_common.h" +#include // printf +#include // intptr_t + + +/*-************************************ +* Local Structures and types +**************************************/ +typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; +typedef enum { full = 0, partial = 1 } earlyEnd_directive; + +#include "lizard_decompress_lz4.h" +#ifndef USE_LZ4_ONLY + #ifdef LIZARD_USE_TEST + #include "test/lizard_common_test.h" + #include "test/lizard_decompress_test.h" + #else + #include "lizard_decompress_liz.h" + #endif +#endif +#include "entropy/huf.h" + + +/*-***************************** +* Decompression functions +*******************************/ + +FORCE_INLINE size_t Lizard_readStream(int flag, const BYTE** ip, const BYTE* const iend, BYTE* op, BYTE* const oend, const BYTE** streamPtr, const BYTE** streamEnd, int streamFlag) +{ + if (!flag) { + if (*ip > iend - 3) return 0; + *streamPtr = *ip + 3; + *streamEnd = *streamPtr + MEM_readLE24(*ip); + if (*streamEnd < *streamPtr) return 0; + *ip = *streamEnd; +#ifdef LIZARD_STATS + uncompr_stream[streamFlag] += *streamEnd-*streamPtr; +#else + (void)streamFlag; +#endif + return 1; + } else { +#ifndef LIZARD_NO_HUFFMAN + size_t res, streamLen, comprStreamLen; + + if (*ip > iend - 6) return 0; + streamLen = MEM_readLE24(*ip); + comprStreamLen = MEM_readLE24(*ip + 3); + + if ((op > oend - streamLen) || (*ip + comprStreamLen > iend - 6)) return 0; + res = HUF_decompress(op, streamLen, *ip + 6, comprStreamLen); + if (HUF_isError(res) || (res != streamLen)) return 0; + + *ip += comprStreamLen + 6; + *streamPtr = op; + *streamEnd = *streamPtr + streamLen; +#ifdef LIZARD_STATS + compr_stream[streamFlag] += comprStreamLen + 6; + decompr_stream[streamFlag] += *streamEnd-*streamPtr; +#endif + return 1; +#else + fprintf(stderr, "compiled with LIZARD_NO_HUFFMAN\n"); + (void)op; (void)oend; + return 0; +#endif + } +} + + +FORCE_INLINE int Lizard_decompress_generic( + const char* source, + char* const dest, + int inputSize, + int outputSize, /* this value is the max size of Output Buffer. */ + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest if dict == noDict */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + /* Local Variables */ + const BYTE* ip = (const BYTE*) source, *istart = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + outputSize; + BYTE* oexit = op + targetOutputSize; + Lizard_parameters params; + Lizard_dstream_t ctx; + BYTE* decompFlagsBase, *decompOff24Base, *decompOff16Base, *decompLiteralsBase = NULL; + int res, compressionLevel; + + if (inputSize < 1) { LIZARD_LOG_DECOMPRESS("inputSize=%d outputSize=%d targetOutputSize=%d partialDecoding=%d\n", inputSize, outputSize, targetOutputSize, partialDecoding); return 0; } + + compressionLevel = *ip++; + + if (compressionLevel < LIZARD_MIN_CLEVEL || compressionLevel > LIZARD_MAX_CLEVEL) { + LIZARD_LOG_DECOMPRESS("ERROR Lizard_decompress_generic inputSize=%d compressionLevel=%d\n", inputSize, compressionLevel); + return -1; + } + + LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic ip=%p inputSize=%d targetOutputSize=%d dest=%p outputSize=%d cLevel=%d dict=%d dictSize=%d dictStart=%p partialDecoding=%d\n", ip, inputSize, targetOutputSize, dest, outputSize, compressionLevel, dict, (int)dictSize, dictStart, partialDecoding); + + decompLiteralsBase = (BYTE*)malloc(4*LIZARD_HUF_BLOCK_SIZE); + if (!decompLiteralsBase) return -1; + decompFlagsBase = decompLiteralsBase + LIZARD_HUF_BLOCK_SIZE; + decompOff24Base = decompFlagsBase + LIZARD_HUF_BLOCK_SIZE; + decompOff16Base = decompOff24Base + LIZARD_HUF_BLOCK_SIZE; + +#ifdef LIZARD_STATS + init_stats(); +#endif + (void)istart; + + while (ip < iend) + { + res = *ip++; + if (res == LIZARD_FLAG_UNCOMPRESSED) /* uncompressed */ + { + uint32_t length; + if (ip > iend - 3) { LIZARD_LOG_DECOMPRESS("UNCOMPRESSED ip[%p] > iend[%p] - 3\n", ip, iend); goto _output_error; } + length = MEM_readLE24(ip); + ip += 3; + // printf("%d: total=%d block=%d UNCOMPRESSED op=%p oexit=%p oend=%p\n", (int)(op-(BYTE*)dest) ,(int)(ip-istart), length, op, oexit, oend); + if (ip + length > iend || op + length > oend) { LIZARD_LOG_DECOMPRESS("UNCOMPRESSED ip[%p]+length[%d] > iend[%p]\n", ip, length, iend); goto _output_error; } + memcpy(op, ip, length); + op += length; + ip += length; + if ((partialDecoding) && (op >= oexit)) break; +#ifdef LIZARD_STATS + uncompr_stream[LIZARD_STREAM_UNCOMPRESSED] += length; +#endif + continue; + } + + if (res&LIZARD_FLAG_LEN) { + LIZARD_LOG_DECOMPRESS("res=%d\n", res); goto _output_error; + } + + if (ip > iend - 5*3) goto _output_error; + ctx.lenPtr = (const BYTE*)ip + 3; + ctx.lenEnd = ctx.lenPtr + MEM_readLE24(ip); + if (ctx.lenEnd < ctx.lenPtr || (ctx.lenEnd > iend - 3)) goto _output_error; +#ifdef LIZARD_STATS + uncompr_stream[LIZARD_STREAM_LEN] += ctx.lenEnd-ctx.lenPtr + 3; +#endif + ip = ctx.lenEnd; + + { size_t streamLen; +#ifdef LIZARD_USE_LOGS + const BYTE* ipos; + size_t comprFlagsLen, comprLiteralsLen, total; +#endif + streamLen = Lizard_readStream(res&LIZARD_FLAG_OFFSET16, &ip, iend, decompOff16Base, decompOff16Base + LIZARD_HUF_BLOCK_SIZE, &ctx.offset16Ptr, &ctx.offset16End, LIZARD_STREAM_OFFSET16); + if (streamLen == 0) goto _output_error; + + streamLen = Lizard_readStream(res&LIZARD_FLAG_OFFSET24, &ip, iend, decompOff24Base, decompOff24Base + LIZARD_HUF_BLOCK_SIZE, &ctx.offset24Ptr, &ctx.offset24End, LIZARD_STREAM_OFFSET24); + if (streamLen == 0) goto _output_error; + +#ifdef LIZARD_USE_LOGS + ipos = ip; + streamLen = Lizard_readStream(res&LIZARD_FLAG_FLAGS, &ip, iend, decompFlagsBase, decompFlagsBase + LIZARD_HUF_BLOCK_SIZE, &ctx.flagsPtr, &ctx.flagsEnd, LIZARD_STREAM_FLAGS); + if (streamLen == 0) goto _output_error; + streamLen = (size_t)(ctx.flagsEnd-ctx.flagsPtr); + comprFlagsLen = ((size_t)(ip - ipos) + 3 >= streamLen) ? 0 : (size_t)(ip - ipos); + ipos = ip; +#else + streamLen = Lizard_readStream(res&LIZARD_FLAG_FLAGS, &ip, iend, decompFlagsBase, decompFlagsBase + LIZARD_HUF_BLOCK_SIZE, &ctx.flagsPtr, &ctx.flagsEnd, LIZARD_STREAM_FLAGS); + if (streamLen == 0) goto _output_error; +#endif + + streamLen = Lizard_readStream(res&LIZARD_FLAG_LITERALS, &ip, iend, decompLiteralsBase, decompLiteralsBase + LIZARD_HUF_BLOCK_SIZE, &ctx.literalsPtr, &ctx.literalsEnd, LIZARD_STREAM_LITERALS); + if (streamLen == 0) goto _output_error; +#ifdef LIZARD_USE_LOGS + streamLen = (size_t)(ctx.literalsEnd-ctx.literalsPtr); + comprLiteralsLen = ((size_t)(ip - ipos) + 3 >= streamLen) ? 0 : (size_t)(ip - ipos); + total = (size_t)(ip-(ctx.lenEnd-1)); +#endif + + if (ip > iend) goto _output_error; + + LIZARD_LOG_DECOMPRESS("%d: total=%d block=%d flagsLen=%d(HUF=%d) literalsLen=%d(HUF=%d) offset16Len=%d offset24Len=%d lengthsLen=%d \n", (int)(op-(BYTE*)dest) ,(int)(ip-istart), (int)total, + (int)(ctx.flagsEnd-ctx.flagsPtr), (int)comprFlagsLen, (int)(ctx.literalsEnd-ctx.literalsPtr), (int)comprLiteralsLen, + (int)(ctx.offset16End-ctx.offset16Ptr), (int)(ctx.offset24End-ctx.offset24Ptr), (int)(ctx.lenEnd-ctx.lenPtr)); + } + + ctx.last_off = -LIZARD_INIT_LAST_OFFSET; + params = Lizard_defaultParameters[compressionLevel - LIZARD_MIN_CLEVEL]; + if (params.decompressType == Lizard_coderwords_LZ4) + res = Lizard_decompress_LZ4(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel); + else +#ifdef USE_LZ4_ONLY + res = Lizard_decompress_LZ4(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel); +#else + res = Lizard_decompress_LIZv1(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel); +#endif + LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic res=%d inputSize=%d\n", res, (int)(ctx.literalsEnd-ctx.lenEnd)); + + if (res <= 0) { free(decompLiteralsBase); return res; } + + op += res; + outputSize -= res; + if ((partialDecoding) && (op >= oexit)) break; + } + +#ifdef LIZARD_STATS + print_stats(); +#endif + + LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic total=%d\n", (int)(op-(BYTE*)dest)); + free(decompLiteralsBase); + return (int)(op-(BYTE*)dest); + +_output_error: + LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic ERROR ip=%p iend=%p\n", ip, iend); + free(decompLiteralsBase); + return -1; +} + + +int Lizard_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) +{ + return Lizard_decompress_generic(source, dest, compressedSize, maxDecompressedSize, full, 0, noDict, (BYTE*)dest, NULL, 0); +} + +int Lizard_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) +{ + return Lizard_decompress_generic(source, dest, compressedSize, maxDecompressedSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); +} + + +/*===== streaming decompression functions =====*/ + + +/* + * If you prefer dynamic allocation methods, + * Lizard_createStreamDecode() + * provides a pointer (void*) towards an initialized Lizard_streamDecode_t structure. + */ +Lizard_streamDecode_t* Lizard_createStreamDecode(void) +{ + Lizard_streamDecode_t* lizards = (Lizard_streamDecode_t*) ALLOCATOR(1, sizeof(Lizard_streamDecode_t)); + return lizards; +} + +int Lizard_freeStreamDecode (Lizard_streamDecode_t* Lizard_stream) +{ + FREEMEM(Lizard_stream); + return 0; +} + +/*! + * Lizard_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * Return : 1 if OK, 0 if error + */ +int Lizard_setStreamDecode (Lizard_streamDecode_t* Lizard_streamDecode, const char* dictionary, int dictSize) +{ + Lizard_streamDecode_t* lizardsd = (Lizard_streamDecode_t*) Lizard_streamDecode; + lizardsd->prefixSize = (size_t) dictSize; + lizardsd->prefixEnd = (const BYTE*) dictionary + dictSize; + lizardsd->externalDict = NULL; + lizardsd->extDictSize = 0; + return 1; +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using Lizard_setStreamDecode() +*/ +int Lizard_decompress_safe_continue (Lizard_streamDecode_t* Lizard_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + Lizard_streamDecode_t* lizardsd = (Lizard_streamDecode_t*) Lizard_streamDecode; + int result; + + if (lizardsd->prefixEnd == (BYTE*)dest) { + result = Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, + full, 0, usingExtDict, lizardsd->prefixEnd - lizardsd->prefixSize, lizardsd->externalDict, lizardsd->extDictSize); + if (result <= 0) return result; + lizardsd->prefixSize += result; + lizardsd->prefixEnd += result; + } else { + lizardsd->extDictSize = lizardsd->prefixSize; + lizardsd->externalDict = lizardsd->prefixEnd - lizardsd->extDictSize; + result = Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, + full, 0, usingExtDict, (BYTE*)dest, lizardsd->externalDict, lizardsd->extDictSize); + if (result <= 0) return result; + lizardsd->prefixSize = result; + lizardsd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +int Lizard_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); + if (dictStart+dictSize == dest) + { + if (dictSize >= (int)(LIZARD_DICT_SIZE - 1)) + return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, withPrefix64k, (BYTE*)dest-LIZARD_DICT_SIZE, NULL, 0); + return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + } + return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +/* debug function */ +int Lizard_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + diff --git a/contrib/lizard/lib/lizard_decompress.h b/contrib/lizard/lib/lizard_decompress.h new file mode 100644 index 00000000000..ad9fc8ee941 --- /dev/null +++ b/contrib/lizard/lib/lizard_decompress.h @@ -0,0 +1,152 @@ +/* + Lizard - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2016, Yann Collet + Copyright (C) 2016-2017, Przemyslaw Skibinski + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ +#ifndef LIZARD_DECOMPRESS_H_2983 +#define LIZARD_DECOMPRESS_H_2983 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/*^*************************************************************** +* Export parameters +*****************************************************************/ +/* +* LIZARD_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(LIZARD_DLL_EXPORT) && (LIZARD_DLL_EXPORT==1) +# define LIZARDDLIB_API __declspec(dllexport) +#elif defined(LIZARD_DLL_IMPORT) && (LIZARD_DLL_IMPORT==1) +# define LIZARDDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define LIZARDDLIB_API +#endif + + +/*-************************************ +* Simple Functions +**************************************/ + +/* +Lizard_decompress_safe() : + compressedSize : is the precise full size of the compressed block. + maxDecompressedSize : is the size of destination buffer, which must be already allocated. + return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) + If destination buffer is not large enough, decoding will stop and output an error code (<0). + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function is protected against buffer overflow exploits, including malicious data packets. + It never writes outside output buffer, nor reads outside input buffer. +*/ +LIZARDDLIB_API int Lizard_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); + + + +/*! +Lizard_decompress_safe_partial() : + This function decompress a compressed block of size 'compressedSize' at position 'source' + into destination buffer 'dest' of size 'maxDecompressedSize'. + The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, + reducing decompression time. + return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) + Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. + Always control how many bytes were decoded. + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets +*/ +LIZARDDLIB_API int Lizard_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); + + + +/*-********************************************** +* Streaming Decompression Functions +************************************************/ +typedef struct Lizard_streamDecode_s Lizard_streamDecode_t; + +/* + * Lizard_streamDecode_t + * information structure to track an Lizard stream. + * init this structure content using Lizard_setStreamDecode or memset() before first use ! + * + * In the context of a DLL (liblizard) please prefer usage of construction methods below. + * They are more future proof, in case of a change of Lizard_streamDecode_t size in the future. + * Lizard_createStreamDecode will allocate and initialize an Lizard_streamDecode_t structure + * Lizard_freeStreamDecode releases its memory. + */ +LIZARDDLIB_API Lizard_streamDecode_t* Lizard_createStreamDecode(void); +LIZARDDLIB_API int Lizard_freeStreamDecode (Lizard_streamDecode_t* Lizard_stream); + +/*! Lizard_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * Setting a size of 0 is allowed (same effect as reset). + * @return : 1 if OK, 0 if error + */ +LIZARDDLIB_API int Lizard_setStreamDecode (Lizard_streamDecode_t* Lizard_streamDecode, const char* dictionary, int dictSize); + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to LIZARD_DICT_SIZE) + In the case of a ring buffers, decoding buffer must be either : + - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + In which case, the decoding & encoding ring buffer can have any size, including small ones ( < LIZARD_DICT_SIZE). + - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including small ones ( < LIZARD_DICT_SIZE). + - _At least_ LIZARD_DICT_SIZE + 8 bytes + maxBlockSize. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including larger than decoding buffer. + Whenever these conditions are not possible, save the last LIZARD_DICT_SIZE of decoded data into a safe buffer, + and indicate where it is saved using Lizard_setStreamDecode() +*/ +LIZARDDLIB_API int Lizard_decompress_safe_continue (Lizard_streamDecode_t* Lizard_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as + a combination of Lizard_setStreamDecode() followed by Lizard_decompress_x_continue() + They are stand-alone. They don't need nor update an Lizard_streamDecode_t structure. +*/ +LIZARDDLIB_API int Lizard_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); + + +#if defined (__cplusplus) +} +#endif + +#endif /* LIZARD_DECOMPRESS_H_2983827168210 */ diff --git a/contrib/lizard/lib/lizard_decompress_liz.h b/contrib/lizard/lib/lizard_decompress_liz.h new file mode 100644 index 00000000000..7e80eb5e01b --- /dev/null +++ b/contrib/lizard/lib/lizard_decompress_liz.h @@ -0,0 +1,220 @@ +/* + [0_MMMM_LLL] - 16-bit offset, 4-bit match length (4-15+), 3-bit literal length (0-7+) + [1_MMMM_LLL] - last offset, 4-bit match length (0-15+), 3-bit literal length (0-7+) + flag 31 - 24-bit offset, match length (47+), no literal length + flag 0-30 - 24-bit offset, 31 match lengths (16-46), no literal length +*/ + +/*! Lizard_decompress_LIZv1() : + * This generic decompression function cover all use cases. + * It shall be instantiated several times, using different sets of directives + * Note that it is important this generic function is really inlined, + * in order to remove useless branches during compilation optimization. + */ +FORCE_INLINE int Lizard_decompress_LIZv1( + Lizard_dstream_t* ctx, + BYTE* const dest, + int outputSize, /* this value is the max size of Output Buffer. */ + + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest if dict == noDict */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize, /* note : = 0 if noDict */ + int compressionLevel + ) +{ + /* Local Variables */ + int inputSize = (int)(ctx->flagsEnd - ctx->flagsPtr); + const BYTE* const blockBase = ctx->flagsPtr; + const BYTE* const iend = ctx->literalsEnd; + + BYTE* op = dest; + BYTE* const oend = op + outputSize; + BYTE* cpy = NULL; + BYTE* oexit = op + targetOutputSize; + const BYTE* const lowLimit = lowPrefix - dictSize; + const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; + + const int checkOffset = (dictSize < (int)(LIZARD_DICT_SIZE)); + + intptr_t last_off = ctx->last_off; + intptr_t length = 0; + (void)compressionLevel; + + /* Special cases */ + if (unlikely(outputSize==0)) return ((inputSize==1) && (*ctx->flagsPtr==0)) ? 0 : -1; /* Empty output buffer */ + + /* Main Loop : decode sequences */ + while (ctx->flagsPtr < ctx->flagsEnd) { + unsigned token; + const BYTE* match; + // intptr_t litLength; + + if ((partialDecoding) && (op >= oexit)) return (int) (op-dest); + + /* get literal length */ + token = *ctx->flagsPtr++; + + if (token >= 32) + { + if ((length=(token & MAX_SHORT_LITLEN)) == MAX_SHORT_LITLEN) { + if (unlikely(ctx->literalsPtr > iend - 1)) { LIZARD_LOG_DECOMPRESS_LIZv1("1"); goto _output_error; } + length = *ctx->literalsPtr; + if unlikely(length >= 254) { + if (length == 254) { + length = MEM_readLE16(ctx->literalsPtr+1); + ctx->literalsPtr += 2; + } else { + length = MEM_readLE24(ctx->literalsPtr+1); + ctx->literalsPtr += 3; + } + } + length += MAX_SHORT_LITLEN; + ctx->literalsPtr++; + if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LIZv1("2"); goto _output_error; } /* overflow detection */ + if (unlikely((size_t)(ctx->literalsPtr+length)<(size_t)(ctx->literalsPtr))) { LIZARD_LOG_DECOMPRESS_LIZv1("3"); goto _output_error; } /* overflow detection */ + } + + /* copy literals */ + cpy = op + length; + if (unlikely(cpy > oend - WILDCOPYLENGTH || ctx->literalsPtr > iend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LIZv1("offset outside buffers\n"); goto _output_error; } /* Error : offset outside buffers */ + #if 1 + Lizard_wildCopy16(op, ctx->literalsPtr, cpy); + op = cpy; + ctx->literalsPtr += length; + #else + Lizard_copy8(op, ctx->literalsPtr); + Lizard_copy8(op+8, ctx->literalsPtr+8); + if (length > 16) + Lizard_wildCopy16(op + 16, ctx->literalsPtr + 16, cpy); + op = cpy; + ctx->literalsPtr += length; + #endif + + /* get offset */ + if (unlikely(ctx->offset16Ptr > ctx->offset16End)) { LIZARD_LOG_DECOMPRESS_LIZv1("(ctx->offset16Ptr > ctx->offset16End\n"); goto _output_error; } +#if 1 + { /* branchless */ + intptr_t new_off = MEM_readLE16(ctx->offset16Ptr); + uintptr_t not_repCode = (uintptr_t)(token >> ML_RUN_BITS) - 1; + last_off ^= not_repCode & (last_off ^ -new_off); + ctx->offset16Ptr = (BYTE*)((uintptr_t)ctx->offset16Ptr + (not_repCode & 2)); + } +#else + if ((token >> ML_RUN_BITS) == 0) + { + last_off = -(intptr_t)MEM_readLE16(ctx->offset16Ptr); + ctx->offset16Ptr += 2; + } +#endif + + /* get matchlength */ + length = (token >> RUN_BITS_LIZv1) & MAX_SHORT_MATCHLEN; + if (length == MAX_SHORT_MATCHLEN) { + if (unlikely(ctx->literalsPtr > iend - 1)) { LIZARD_LOG_DECOMPRESS_LIZv1("6"); goto _output_error; } + length = *ctx->literalsPtr; + if unlikely(length >= 254) { + if (length == 254) { + length = MEM_readLE16(ctx->literalsPtr+1); + ctx->literalsPtr += 2; + } else { + length = MEM_readLE24(ctx->literalsPtr+1); + ctx->literalsPtr += 3; + } + } + length += MAX_SHORT_MATCHLEN; + ctx->literalsPtr++; + if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LIZv1("7"); goto _output_error; } /* overflow detection */ + } + + DECOMPLOG_CODEWORDS_LIZv1("T32+ literal=%u match=%u offset=%d ipos=%d opos=%d\n", (U32)litLength, (U32)length, (int)-last_off, (U32)(ctx->flagsPtr-blockBase), (U32)(op-dest)); + } + else + if (token < LIZARD_LAST_LONG_OFF) + { + if (unlikely(ctx->offset24Ptr > ctx->offset24End - 3)) { LIZARD_LOG_DECOMPRESS_LIZv1("8"); goto _output_error; } + length = token + MM_LONGOFF; + last_off = -(intptr_t)MEM_readLE24(ctx->offset24Ptr); + ctx->offset24Ptr += 3; + DECOMPLOG_CODEWORDS_LIZv1("T0-30 literal=%u match=%u offset=%d\n", 0, (U32)length, (int)-last_off); + } + else + { + if (unlikely(ctx->literalsPtr > iend - 1)) { LIZARD_LOG_DECOMPRESS_LIZv1("9"); goto _output_error; } + length = *ctx->literalsPtr; + if unlikely(length >= 254) { + if (length == 254) { + length = MEM_readLE16(ctx->literalsPtr+1); + ctx->literalsPtr += 2; + } else { + length = MEM_readLE24(ctx->literalsPtr+1); + ctx->literalsPtr += 3; + } + } + ctx->literalsPtr++; + length += LIZARD_LAST_LONG_OFF + MM_LONGOFF; + + if (unlikely(ctx->offset24Ptr > ctx->offset24End - 3)) { LIZARD_LOG_DECOMPRESS_LIZv1("10"); goto _output_error; } + last_off = -(intptr_t)MEM_readLE24(ctx->offset24Ptr); + ctx->offset24Ptr += 3; + } + + + match = op + last_off; + if ((checkOffset) && ((unlikely((uintptr_t)(-last_off) > (uintptr_t)op) || (match < lowLimit)))) { LIZARD_LOG_DECOMPRESS_LIZv1("lowPrefix[%p]-dictSize[%d]=lowLimit[%p] match[%p]=op[%p]-last_off[%d]\n", lowPrefix, (int)dictSize, lowLimit, match, op, (int)last_off); goto _output_error; } /* Error : offset outside buffers */ + + /* check external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op + length > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LIZv1("12"); goto _output_error; } /* doesn't respect parsing restriction */ + + if (length <= (intptr_t)(lowPrefix - match)) { + /* match can be copied as a single segment from external dictionary */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match encompass external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix-match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + if (unlikely(cpy > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LIZv1("13match=%p lowLimit=%p\n", match, lowLimit); goto _output_error; } /* Error : offset outside buffers */ + Lizard_copy8(op, match); + Lizard_copy8(op+8, match+8); + if (length > 16) + Lizard_wildCopy16(op + 16, match + 16, cpy); + op = cpy; + } + + /* last literals */ + length = ctx->literalsEnd - ctx->literalsPtr; + cpy = op + length; + if ((length < 0) || (ctx->literalsPtr+length != iend) || (cpy > oend)) { LIZARD_LOG_DECOMPRESS_LIZv1("14"); goto _output_error; } /* Error : input must be consumed */ + memcpy(op, ctx->literalsPtr, length); + ctx->literalsPtr += length; + op += length; + + /* end of decoding */ + ctx->last_off = last_off; + return (int) (op-dest); /* Nb of output bytes decoded */ + + /* Overflow error detected */ +_output_error: + LIZARD_LOG_DECOMPRESS_LIZv1("_output_error=%d ctx->flagsPtr=%p blockBase=%p\n", (int) (-(ctx->flagsPtr-blockBase))-1, ctx->flagsPtr, blockBase); + LIZARD_LOG_DECOMPRESS_LIZv1("cpy=%p oend=%p ctx->literalsPtr+length[%d]=%p iend=%p\n", cpy, oend, (int)length, ctx->literalsPtr+length, iend); + return (int) (-(ctx->flagsPtr-blockBase))-1; +} diff --git a/contrib/lizard/lib/lizard_decompress_lz4.h b/contrib/lizard/lib/lizard_decompress_lz4.h new file mode 100644 index 00000000000..ffbef1cd976 --- /dev/null +++ b/contrib/lizard/lib/lizard_decompress_lz4.h @@ -0,0 +1,163 @@ +/*! Lizard_decompress_LZ4() : + * This generic decompression function cover all use cases. + * It shall be instantiated several times, using different sets of directives + * Note that it is important this generic function is really inlined, + * in order to remove useless branches during compilation optimization. + */ +FORCE_INLINE int Lizard_decompress_LZ4( + Lizard_dstream_t* ctx, + BYTE* const dest, + int outputSize, /* this value is the max size of Output Buffer. */ + + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest if dict == noDict */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize, /* note : = 0 if noDict */ + int compressionLevel + ) +{ + /* Local Variables */ + int inputSize = (int)(ctx->flagsEnd - ctx->flagsPtr); + const BYTE* const blockBase = ctx->flagsPtr; + const BYTE* const iend = ctx->literalsEnd; + BYTE* op = dest; + BYTE* const oend = op + outputSize; + BYTE* cpy = NULL; + BYTE* oexit = op + targetOutputSize; + const BYTE* const lowLimit = lowPrefix - dictSize; + const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; + + const int checkOffset = (dictSize < (int)(LIZARD_DICT_SIZE)); + + intptr_t length = 0; + (void)compressionLevel; + + /* Special cases */ + if (unlikely(outputSize==0)) return ((inputSize==1) && (*ctx->flagsPtr==0)) ? 0 : -1; /* Empty output buffer */ + + /* Main Loop : decode sequences */ + while (ctx->flagsPtr < ctx->flagsEnd) { + unsigned token; + const BYTE* match; + size_t offset; + + /* get literal length */ + token = *ctx->flagsPtr++; + if ((length=(token & RUN_MASK_LZ4)) == RUN_MASK_LZ4) { + if (unlikely(ctx->literalsPtr > iend - 5)) { LIZARD_LOG_DECOMPRESS_LZ4("0"); goto _output_error; } + length = *ctx->literalsPtr; + if unlikely(length >= 254) { + if (length == 254) { + length = MEM_readLE16(ctx->literalsPtr+1); + ctx->literalsPtr += 2; + } else { + length = MEM_readLE24(ctx->literalsPtr+1); + ctx->literalsPtr += 3; + } + } + length += RUN_MASK_LZ4; + ctx->literalsPtr++; + if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LZ4("1"); goto _output_error; } /* overflow detection */ + if (unlikely((size_t)(ctx->literalsPtr+length)<(size_t)(ctx->literalsPtr))) { LIZARD_LOG_DECOMPRESS_LZ4("2"); goto _output_error; } /* overflow detection */ + } + + /* copy literals */ + cpy = op + length; + if (unlikely(cpy > oend - WILDCOPYLENGTH || ctx->literalsPtr + length > iend - (2 + WILDCOPYLENGTH))) { LIZARD_LOG_DECOMPRESS_LZ4("offset outside buffers\n"); goto _output_error; } /* Error : offset outside buffers */ + +#if 1 + Lizard_wildCopy16(op, ctx->literalsPtr, cpy); + op = cpy; + ctx->literalsPtr += length; +#else + Lizard_copy8(op, ctx->literalsPtr); + Lizard_copy8(op+8, ctx->literalsPtr+8); + if (length > 16) + Lizard_wildCopy16(op + 16, ctx->literalsPtr + 16, cpy); + op = cpy; + ctx->literalsPtr += length; +#endif + if ((partialDecoding) && (op >= oexit)) return (int) (op-dest); + + /* get offset */ + offset = MEM_readLE16(ctx->literalsPtr); + ctx->literalsPtr += 2; + + match = op - offset; + if ((checkOffset) && (unlikely(match < lowLimit))) { LIZARD_LOG_DECOMPRESS_LZ4("lowPrefix[%p]-dictSize[%d]=lowLimit[%p] match[%p]=op[%p]-offset[%d]\n", lowPrefix, (int)dictSize, lowLimit, match, op, (int)offset); goto _output_error; } /* Error : offset outside buffers */ + + /* get matchlength */ + length = token >> RUN_BITS_LZ4; + if (length == ML_MASK_LZ4) { + if (unlikely(ctx->literalsPtr > iend - 5)) { LIZARD_LOG_DECOMPRESS_LZ4("4"); goto _output_error; } + length = *ctx->literalsPtr; + if unlikely(length >= 254) { + if (length == 254) { + length = MEM_readLE16(ctx->literalsPtr+1); + ctx->literalsPtr += 2; + } else { + length = MEM_readLE24(ctx->literalsPtr+1); + ctx->literalsPtr += 3; + } + } + length += ML_MASK_LZ4; + ctx->literalsPtr++; + if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LZ4("5"); goto _output_error; } /* overflow detection */ + } + length += MINMATCH; + + /* check external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op + length > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LZ4("6"); goto _output_error; } /* doesn't respect parsing restriction */ + + if (length <= (intptr_t)(lowPrefix - match)) { + /* match can be copied as a single segment from external dictionary */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match encompass external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix-match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + if (unlikely(cpy > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LZ4("1match=%p lowLimit=%p\n", match, lowLimit); goto _output_error; } /* Error : offset outside buffers */ + Lizard_copy8(op, match); + Lizard_copy8(op+8, match+8); + if (length > 16) + Lizard_wildCopy16(op + 16, match + 16, cpy); + op = cpy; + if ((partialDecoding) && (op >= oexit)) return (int) (op-dest); + } + + /* last literals */ + length = ctx->literalsEnd - ctx->literalsPtr; + cpy = op + length; + if ((length < 0) || (ctx->literalsPtr+length != iend) || (cpy > oend)) { LIZARD_LOG_DECOMPRESS_LZ4("9"); goto _output_error; } /* Error : input must be consumed */ + memcpy(op, ctx->literalsPtr, length); + ctx->literalsPtr += length; + op += length; + + /* end of decoding */ + return (int) (op-dest); /* Nb of output bytes decoded */ + + /* Overflow error detected */ +_output_error: + LIZARD_LOG_DECOMPRESS_LZ4("_output_error=%d ctx->flagsPtr=%p blockBase=%p\n", (int) (-(ctx->flagsPtr-blockBase))-1, ctx->flagsPtr, blockBase); + LIZARD_LOG_DECOMPRESS_LZ4("cpy=%p oend=%p ctx->literalsPtr+length[%d]=%p iend=%p\n", cpy, oend, (int)length, ctx->literalsPtr+length, iend); + return (int) (-(ctx->flagsPtr-blockBase))-1; +} diff --git a/contrib/lizard/lib/lizard_frame.c b/contrib/lizard/lib/lizard_frame.c new file mode 100644 index 00000000000..f4afbb94e0a --- /dev/null +++ b/contrib/lizard/lib/lizard_frame.c @@ -0,0 +1,1362 @@ +/* +Lizard auto-framing library +Copyright (C) 2011-2016, Yann Collet +Copyright (C) 2016-2017, Przemyslaw Skibinski + +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- Lizard source repository : https://github.com/inikep/lizard +*/ + +/* LizardF is a stand-alone API to create Lizard-compressed Frames +* in full conformance with specification v1.5.0 +* All related operations, including memory management, are handled by the library. +* */ + + +/*-************************************ +* Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + + +/*-************************************ +* Includes +**************************************/ +#include "lizard_frame_static.h" +#include "lizard_compress.h" +#include "lizard_decompress.h" +#include "lizard_common.h" /* LIZARD_DICT_SIZE */ +#define XXH_STATIC_LINKING_ONLY +#include "xxhash/xxhash.h" +#include + + + +/* unoptimized version; solves endianess & alignment issues */ +static U32 LizardF_readLE32 (const void* src) +{ + const BYTE* const srcPtr = (const BYTE*)src; + U32 value32 = srcPtr[0]; + value32 += (srcPtr[1]<<8); + value32 += (srcPtr[2]<<16); + value32 += ((U32)srcPtr[3])<<24; + return value32; +} + +static void LizardF_writeLE32 (BYTE* dstPtr, U32 value32) +{ + dstPtr[0] = (BYTE)value32; + dstPtr[1] = (BYTE)(value32 >> 8); + dstPtr[2] = (BYTE)(value32 >> 16); + dstPtr[3] = (BYTE)(value32 >> 24); +} + +static U64 LizardF_readLE64 (const BYTE* srcPtr) +{ + U64 value64 = srcPtr[0]; + value64 += ((U64)srcPtr[1]<<8); + value64 += ((U64)srcPtr[2]<<16); + value64 += ((U64)srcPtr[3]<<24); + value64 += ((U64)srcPtr[4]<<32); + value64 += ((U64)srcPtr[5]<<40); + value64 += ((U64)srcPtr[6]<<48); + value64 += ((U64)srcPtr[7]<<56); + return value64; +} + +static void LizardF_writeLE64 (BYTE* dstPtr, U64 value64) +{ + dstPtr[0] = (BYTE)value64; + dstPtr[1] = (BYTE)(value64 >> 8); + dstPtr[2] = (BYTE)(value64 >> 16); + dstPtr[3] = (BYTE)(value64 >> 24); + dstPtr[4] = (BYTE)(value64 >> 32); + dstPtr[5] = (BYTE)(value64 >> 40); + dstPtr[6] = (BYTE)(value64 >> 48); + dstPtr[7] = (BYTE)(value64 >> 56); +} + + +/*-************************************ +* Constants +**************************************/ +#define _1BIT 0x01 +#define _2BITS 0x03 +#define _3BITS 0x07 +#define _4BITS 0x0F +#define _8BITS 0xFF + +#define LIZARDF_MAGIC_SKIPPABLE_START 0x184D2A50U +#define LIZARDF_MAGICNUMBER 0x184D2206U +#define LIZARDF_BLOCKUNCOMPRESSED_FLAG 0x80000000U +#define LIZARDF_BLOCKSIZEID_DEFAULT LizardF_max128KB + +static const size_t minFHSize = 7; +static const size_t maxFHSize = 15; +static const size_t BHSize = 4; + + +/*-************************************ +* Structures and local types +**************************************/ +typedef struct LizardF_cctx_s +{ + LizardF_preferences_t prefs; + U32 version; + U32 cStage; + size_t maxBlockSize; + size_t maxBufferSize; + BYTE* tmpBuff; + BYTE* tmpIn; + size_t tmpInSize; + U64 totalInSize; + XXH32_state_t xxh; + Lizard_stream_t* lizardCtxPtr; + U32 lizardCtxLevel; /* 0: unallocated; 1: Lizard_stream_t; */ +} LizardF_cctx_t; + +typedef struct LizardF_dctx_s +{ + LizardF_frameInfo_t frameInfo; + U32 version; + U32 dStage; + U64 frameRemainingSize; + size_t maxBlockSize; + size_t maxBufferSize; + const BYTE* srcExpect; + BYTE* tmpIn; + size_t tmpInSize; + size_t tmpInTarget; + BYTE* tmpOutBuffer; + const BYTE* dict; + size_t dictSize; + BYTE* tmpOut; + size_t tmpOutSize; + size_t tmpOutStart; + XXH32_state_t xxh; + BYTE header[16]; +} LizardF_dctx_t; + + +/*-************************************ +* Error management +**************************************/ +#define LIZARDF_GENERATE_STRING(STRING) #STRING, +static const char* LizardF_errorStrings[] = { LIZARDF_LIST_ERRORS(LIZARDF_GENERATE_STRING) }; + + +unsigned LizardF_isError(LizardF_errorCode_t code) +{ + return (code > (LizardF_errorCode_t)(-LizardF_ERROR_maxCode)); +} + +const char* LizardF_getErrorName(LizardF_errorCode_t code) +{ + static const char* codeError = "Unspecified error code"; + if (LizardF_isError(code)) return LizardF_errorStrings[-(int)(code)]; + return codeError; +} + + +/*-************************************ +* Private functions +**************************************/ +static size_t LizardF_getBlockSize(unsigned blockSizeID) +{ + static const size_t blockSizes[7] = { 128 KB, 256 KB, 1 MB, 4 MB, 16 MB, 64 MB, 256 MB }; + + if (blockSizeID == 0) blockSizeID = LIZARDF_BLOCKSIZEID_DEFAULT; + blockSizeID -= 1; + if (blockSizeID >= 7) return (size_t)-LizardF_ERROR_maxBlockSize_invalid; + + return blockSizes[blockSizeID]; +} + +static LizardF_blockSizeID_t LizardF_optimalBSID(const LizardF_blockSizeID_t requestedBSID, const size_t srcSize) +{ + LizardF_blockSizeID_t proposedBSID = LizardF_max128KB; + size_t maxBlockSize; + + while (requestedBSID > proposedBSID) + { + maxBlockSize = LizardF_getBlockSize(proposedBSID); + if (srcSize <= maxBlockSize) { + return proposedBSID; + } + proposedBSID = (LizardF_blockSizeID_t)((int)proposedBSID + 1); + } + return requestedBSID; +} + + +static BYTE LizardF_headerChecksum (const void* header, size_t length) +{ + U32 xxh = XXH32(header, length, 0); + return (BYTE)(xxh >> 8); +} + + +/*-************************************ +* Simple compression functions +**************************************/ + +size_t LizardF_compressFrameBound(size_t srcSize, const LizardF_preferences_t* preferencesPtr) +{ + LizardF_preferences_t prefs; + size_t headerSize; + size_t streamSize; + + if (preferencesPtr!=NULL) prefs = *preferencesPtr; + else memset(&prefs, 0, sizeof(prefs)); + + prefs.frameInfo.blockSizeID = LizardF_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); + prefs.autoFlush = 1; + + headerSize = maxFHSize; /* header size, including magic number and frame content size*/ + streamSize = LizardF_compressBound(srcSize, &prefs); + + return headerSize + streamSize; +} + + + +/*! LizardF_compressFrame() : +* Compress an entire srcBuffer into a valid Lizard frame, as defined by specification v1.5.0, in a single step. +* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. +* You can get the minimum value of dstMaxSize by using LizardF_compressFrameBound() +* If this condition is not respected, LizardF_compressFrame() will fail (result is an errorCode) +* The LizardF_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default. +* The result of the function is the number of bytes written into dstBuffer. +* The function outputs an error code if it fails (can be tested using LizardF_isError()) +*/ +size_t LizardF_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_preferences_t* preferencesPtr) +{ + LizardF_cctx_t cctxI; + LizardF_preferences_t prefs; + LizardF_compressOptions_t options; + LizardF_errorCode_t errorCode; + BYTE* const dstStart = (BYTE*) dstBuffer; + BYTE* dstPtr = dstStart; + BYTE* const dstEnd = dstStart + dstMaxSize; + + memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */ + memset(&options, 0, sizeof(options)); + + cctxI.version = LIZARDF_VERSION; + cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ + + if (preferencesPtr!=NULL) + prefs = *preferencesPtr; + else + memset(&prefs, 0, sizeof(prefs)); + if (prefs.frameInfo.contentSize != 0) + prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ + + prefs.frameInfo.blockSizeID = LizardF_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); + prefs.autoFlush = 1; + if (srcSize <= LizardF_getBlockSize(prefs.frameInfo.blockSizeID)) + prefs.frameInfo.blockMode = LizardF_blockIndependent; /* no need for linked blocks */ + + options.stableSrc = 1; + + if (dstMaxSize < LizardF_compressFrameBound(srcSize, &prefs)) + return (size_t)-LizardF_ERROR_dstMaxSize_tooSmall; + + errorCode = LizardF_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */ + if (LizardF_isError(errorCode)) goto error; + dstPtr += errorCode; /* header size */ + + errorCode = LizardF_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); + if (LizardF_isError(errorCode)) goto error; + dstPtr += errorCode; + + errorCode = LizardF_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ + if (LizardF_isError(errorCode)) goto error; + dstPtr += errorCode; + + Lizard_freeStream(cctxI.lizardCtxPtr); + FREEMEM(cctxI.tmpBuff); + return (dstPtr - dstStart); +error: + Lizard_freeStream(cctxI.lizardCtxPtr); + FREEMEM(cctxI.tmpBuff); + return errorCode; +} + + +/*-********************************* +* Advanced compression functions +***********************************/ + +/* LizardF_createCompressionContext() : +* The first thing to do is to create a compressionContext object, which will be used in all compression operations. +* This is achieved using LizardF_createCompressionContext(), which takes as argument a version and an LizardF_preferences_t structure. +* The version provided MUST be LIZARDF_VERSION. It is intended to track potential version differences between different binaries. +* The function will provide a pointer to an allocated LizardF_compressionContext_t object. +* If the result LizardF_errorCode_t is not OK_NoError, there was an error during context creation. +* Object can release its memory using LizardF_freeCompressionContext(); +*/ +LizardF_errorCode_t LizardF_createCompressionContext(LizardF_compressionContext_t* LizardF_compressionContextPtr, unsigned version) +{ + LizardF_cctx_t* cctxPtr; + + cctxPtr = (LizardF_cctx_t*)ALLOCATOR(1, sizeof(LizardF_cctx_t)); + if (cctxPtr==NULL) return (LizardF_errorCode_t)(-LizardF_ERROR_allocation_failed); + + cctxPtr->version = version; + cctxPtr->cStage = 0; /* Next stage : write header */ + + *LizardF_compressionContextPtr = (LizardF_compressionContext_t)cctxPtr; + + return LizardF_OK_NoError; +} + + +LizardF_errorCode_t LizardF_freeCompressionContext(LizardF_compressionContext_t LizardF_compressionContext) +{ + LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)LizardF_compressionContext; + + if (cctxPtr != NULL) { /* null pointers can be safely provided to this function, like free() */ + Lizard_freeStream(cctxPtr->lizardCtxPtr); + FREEMEM(cctxPtr->tmpBuff); + FREEMEM(LizardF_compressionContext); + } + + return LizardF_OK_NoError; +} + + +/*! LizardF_compressBegin() : +* will write the frame header into dstBuffer. +* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LizardF_MAXHEADERFRAME_SIZE bytes. +* The result of the function is the number of bytes written into dstBuffer for the header +* or an error code (can be tested using LizardF_isError()) +*/ +size_t LizardF_compressBegin(LizardF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LizardF_preferences_t* preferencesPtr) +{ + LizardF_preferences_t prefNull; + LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + BYTE* headerStart; + size_t requiredBuffSize; + + if (dstMaxSize < maxFHSize) return (size_t)-LizardF_ERROR_dstMaxSize_tooSmall; + if (cctxPtr->cStage != 0) return (size_t)-LizardF_ERROR_GENERIC; + memset(&prefNull, 0, sizeof(prefNull)); + if (preferencesPtr == NULL) preferencesPtr = &prefNull; + cctxPtr->prefs = *preferencesPtr; + + /* ctx Management */ + if (cctxPtr->lizardCtxLevel == 0) { + cctxPtr->lizardCtxPtr = Lizard_createStream(cctxPtr->prefs.compressionLevel); + cctxPtr->lizardCtxLevel = 1; + } + + /* Buffer Management */ + if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LIZARDF_BLOCKSIZEID_DEFAULT; + cctxPtr->maxBlockSize = LizardF_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); + requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LizardF_blockLinked) * 2 * LIZARD_DICT_SIZE); + + if (preferencesPtr->autoFlush) + requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LizardF_blockLinked) * LIZARD_DICT_SIZE; /* just needs dict */ + + if (cctxPtr->maxBufferSize < requiredBuffSize) { + cctxPtr->maxBufferSize = requiredBuffSize; + FREEMEM(cctxPtr->tmpBuff); + cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(1, requiredBuffSize); + if (cctxPtr->tmpBuff == NULL) { printf("ERROR in LizardF_compressBegin: Cannot allocate %d MB\n", (int)(requiredBuffSize>>20)); return (size_t)-LizardF_ERROR_allocation_failed; } + } + cctxPtr->tmpIn = cctxPtr->tmpBuff; + cctxPtr->tmpInSize = 0; + XXH32_reset(&(cctxPtr->xxh), 0); + cctxPtr->lizardCtxPtr = Lizard_resetStream((Lizard_stream_t*)(cctxPtr->lizardCtxPtr), cctxPtr->prefs.compressionLevel); + if (!cctxPtr->lizardCtxPtr) return (size_t)-LizardF_ERROR_allocation_failed; + + /* Magic Number */ + LizardF_writeLE32(dstPtr, LIZARDF_MAGICNUMBER); + dstPtr += 4; + headerStart = dstPtr; + + /* FLG Byte */ + *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ + + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ + + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */ + + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)); /* Frame content size */ + /* BD Byte */ + *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); + /* Optional Frame content size field */ + if (cctxPtr->prefs.frameInfo.contentSize) { + LizardF_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize); + dstPtr += 8; + cctxPtr->totalInSize = 0; + } + /* CRC Byte */ + *dstPtr = LizardF_headerChecksum(headerStart, dstPtr - headerStart); + dstPtr++; + + cctxPtr->cStage = 1; /* header written, now request input data block */ + + return (dstPtr - dstStart); +} + + +/* LizardF_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. +* The LizardF_frameInfo_t structure is optional : +* you can provide NULL as argument, preferences will then be set to cover worst case situations. +* */ +size_t LizardF_compressBound(size_t srcSize, const LizardF_preferences_t* preferencesPtr) +{ + LizardF_preferences_t prefsNull; + memset(&prefsNull, 0, sizeof(prefsNull)); + prefsNull.frameInfo.contentChecksumFlag = LizardF_contentChecksumEnabled; /* worst case */ + { const LizardF_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + LizardF_blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; + size_t blockSize = LizardF_getBlockSize(bid); + unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; + size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; + size_t blockInfo = 4; /* default, without block CRC option */ + size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); + + return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; + } +} + + +typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level); + +static size_t LizardF_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lizardctx, int level) +{ + /* compress one block */ + BYTE* cSizePtr = (BYTE*)dst; + U32 cSize; + cSize = (U32)compress(lizardctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); + LizardF_writeLE32(cSizePtr, cSize); + if (cSize == 0) { /* compression failed */ + cSize = (U32)srcSize; + LizardF_writeLE32(cSizePtr, cSize + LIZARDF_BLOCKUNCOMPRESSED_FLAG); + memcpy(cSizePtr+4, src, srcSize); + } + return cSize + 4; +} + + + +static int LizardF_localLizard_compress_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +{ + (void)level; + return Lizard_compress_continue((Lizard_stream_t*)ctx, src, dst, srcSize, dstSize); +} + +static compressFunc_t LizardF_selectCompression(LizardF_blockMode_t blockMode) +{ + if (blockMode == LizardF_blockIndependent) return Lizard_compress_extState; + return LizardF_localLizard_compress_continue; +} + +static int LizardF_localSaveDict(LizardF_cctx_t* cctxPtr) +{ + return Lizard_saveDict ((Lizard_stream_t*)(cctxPtr->lizardCtxPtr), (char*)(cctxPtr->tmpBuff), LIZARD_DICT_SIZE); +} + +typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LizardF_lastBlockStatus; + +/*! LizardF_compressUpdate() : +* LizardF_compressUpdate() can be called repetitively to compress as much data as necessary. +* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. +* If this condition is not respected, LizardF_compress() will fail (result is an errorCode) +* You can get the minimum value of dstMaxSize by using LizardF_compressBound() +* The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. +* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. +* The function outputs an error code if it fails (can be tested using LizardF_isError()) +*/ +size_t LizardF_compressUpdate(LizardF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_compressOptions_t* compressOptionsPtr) +{ + LizardF_compressOptions_t cOptionsNull; + LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)compressionContext; + size_t blockSize = cctxPtr->maxBlockSize; + const BYTE* srcPtr = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcPtr + srcSize; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + LizardF_lastBlockStatus lastBlockCompressed = notDone; + compressFunc_t compress; + + + if (cctxPtr->cStage != 1) return (size_t)-LizardF_ERROR_GENERIC; + if (dstMaxSize < LizardF_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-LizardF_ERROR_dstMaxSize_tooSmall; + memset(&cOptionsNull, 0, sizeof(cOptionsNull)); + if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; + + /* select compression function */ + compress = LizardF_selectCompression(cctxPtr->prefs.frameInfo.blockMode); + + /* complete tmp buffer */ + if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */ + size_t sizeToCopy = blockSize - cctxPtr->tmpInSize; + if (sizeToCopy > srcSize) { + /* add src to tmpIn buffer */ + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); + srcPtr = srcEnd; + cctxPtr->tmpInSize += srcSize; + /* still needs some CRC */ + } else { + /* complete tmpIn block and then compress it */ + lastBlockCompressed = fromTmpBuffer; + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); + srcPtr += sizeToCopy; + + dstPtr += LizardF_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lizardCtxPtr, cctxPtr->prefs.compressionLevel); + + if (cctxPtr->prefs.frameInfo.blockMode==LizardF_blockLinked) cctxPtr->tmpIn += blockSize; + cctxPtr->tmpInSize = 0; + } + } + + while ((size_t)(srcEnd - srcPtr) >= blockSize) { + /* compress full block */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LizardF_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lizardCtxPtr, cctxPtr->prefs.compressionLevel); + srcPtr += blockSize; + } + + if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { + /* compress remaining input < blockSize */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LizardF_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lizardCtxPtr, cctxPtr->prefs.compressionLevel); + srcPtr = srcEnd; + } + + /* preserve dictionary if necessary */ + if ((cctxPtr->prefs.frameInfo.blockMode==LizardF_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { + if (compressOptionsPtr->stableSrc) { + cctxPtr->tmpIn = cctxPtr->tmpBuff; + } else { + int realDictSize = LizardF_localSaveDict(cctxPtr); + if (realDictSize==0) return (size_t)-LizardF_ERROR_GENERIC; + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + } + + /* keep tmpIn within limits */ + if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LizardF_blockLinked && lastBlockCompressed==fromTmpBuffer */ + && !(cctxPtr->prefs.autoFlush)) + { + int realDictSize = LizardF_localSaveDict(cctxPtr); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + + /* some input data left, necessarily < blockSize */ + if (srcPtr < srcEnd) { + /* fill tmp buffer */ + size_t sizeToCopy = srcEnd - srcPtr; + memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); + cctxPtr->tmpInSize = sizeToCopy; + } + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LizardF_contentChecksumEnabled) + XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); + + cctxPtr->totalInSize += srcSize; + return dstPtr - dstStart; +} + + +/*! LizardF_flush() : +* Should you need to create compressed data immediately, without waiting for a block to be filled, +* you can call Lizard_flush(), which will immediately compress any remaining data stored within compressionContext. +* The result of the function is the number of bytes written into dstBuffer +* (it can be zero, this means there was no data left within compressionContext) +* The function outputs an error code if it fails (can be tested using LizardF_isError()) +* The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. +*/ +size_t LizardF_flush(LizardF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* compressOptionsPtr) +{ + LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + compressFunc_t compress; + + + if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ + if (cctxPtr->cStage != 1) return (size_t)-LizardF_ERROR_GENERIC; + if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return (size_t)-LizardF_ERROR_dstMaxSize_tooSmall; /* +8 : block header(4) + block checksum(4) */ + (void)compressOptionsPtr; /* not yet useful */ + + /* select compression function */ + compress = LizardF_selectCompression(cctxPtr->prefs.frameInfo.blockMode); + + /* compress tmp buffer */ + dstPtr += LizardF_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lizardCtxPtr, cctxPtr->prefs.compressionLevel); + if (cctxPtr->prefs.frameInfo.blockMode==LizardF_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; + cctxPtr->tmpInSize = 0; + + /* keep tmpIn within limits */ + if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LizardF_blockLinked */ + int realDictSize = LizardF_localSaveDict(cctxPtr); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + + return dstPtr - dstStart; +} + + +/*! LizardF_compressEnd() : +* When you want to properly finish the compressed frame, just call LizardF_compressEnd(). +* It will flush whatever data remained within compressionContext (like Lizard_flush()) +* but also properly finalize the frame, with an endMark and a checksum. +* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) +* The function outputs an error code if it fails (can be tested using LizardF_isError()) +* The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. +* compressionContext can then be used again, starting with LizardF_compressBegin(). The preferences will remain the same. +*/ +size_t LizardF_compressEnd(LizardF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* compressOptionsPtr) +{ + LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + size_t errorCode; + + errorCode = LizardF_flush(compressionContext, dstBuffer, dstMaxSize, compressOptionsPtr); + if (LizardF_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + LizardF_writeLE32(dstPtr, 0); + dstPtr+=4; /* endMark */ + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LizardF_contentChecksumEnabled) { + U32 xxh = XXH32_digest(&(cctxPtr->xxh)); + LizardF_writeLE32(dstPtr, xxh); + dstPtr+=4; /* content Checksum */ + } + + cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ + cctxPtr->maxBufferSize = 0; /* reuse HC context */ + + if (cctxPtr->prefs.frameInfo.contentSize) { + if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize) + return (size_t)-LizardF_ERROR_frameSize_wrong; + } + + return dstPtr - dstStart; +} + + +/*-*************************************************** +* Frame Decompression +*****************************************************/ + +/* Resource management */ + +/*! LizardF_createDecompressionContext() : +* Create a decompressionContext object, which will track all decompression operations. +* Provides a pointer to a fully allocated and initialized LizardF_decompressionContext object. +* Object can later be released using LizardF_freeDecompressionContext(). +* @return : if != 0, there was an error during context creation. +*/ +LizardF_errorCode_t LizardF_createDecompressionContext(LizardF_decompressionContext_t* LizardF_decompressionContextPtr, unsigned versionNumber) +{ + LizardF_dctx_t* const dctxPtr = (LizardF_dctx_t*)ALLOCATOR(1, sizeof(LizardF_dctx_t)); + if (dctxPtr==NULL) return (LizardF_errorCode_t)-LizardF_ERROR_GENERIC; + + dctxPtr->version = versionNumber; + *LizardF_decompressionContextPtr = (LizardF_decompressionContext_t)dctxPtr; + return LizardF_OK_NoError; +} + +LizardF_errorCode_t LizardF_freeDecompressionContext(LizardF_decompressionContext_t LizardF_decompressionContext) +{ + LizardF_errorCode_t result = LizardF_OK_NoError; + LizardF_dctx_t* const dctxPtr = (LizardF_dctx_t*)LizardF_decompressionContext; + if (dctxPtr != NULL) { /* can accept NULL input, like free() */ + result = (LizardF_errorCode_t)dctxPtr->dStage; + FREEMEM(dctxPtr->tmpIn); + FREEMEM(dctxPtr->tmpOutBuffer); + FREEMEM(dctxPtr); + } + return result; +} + + +/* ******************************************************************** */ +/* ********************* Decompression ******************************** */ +/* ******************************************************************** */ + +typedef enum { dstage_getHeader=0, dstage_storeHeader, + dstage_getCBlockSize, dstage_storeCBlockSize, + dstage_copyDirect, + dstage_getCBlock, dstage_storeCBlock, + dstage_decodeCBlock, dstage_decodeCBlock_intoDst, + dstage_decodeCBlock_intoTmp, dstage_flushOut, + dstage_getSuffix, dstage_storeSuffix, + dstage_getSFrameSize, dstage_storeSFrameSize, + dstage_skipSkippable +} dStage_t; + + +/*! LizardF_headerSize() : +* @return : size of frame header +* or an error code, which can be tested using LizardF_isError() +*/ +static size_t LizardF_headerSize(const void* src, size_t srcSize) +{ + /* minimal srcSize to determine header size */ + if (srcSize < 5) return (size_t)-LizardF_ERROR_frameHeader_incomplete; + + /* special case : skippable frames */ + if ((LizardF_readLE32(src) & 0xFFFFFFF0U) == LIZARDF_MAGIC_SKIPPABLE_START) return 8; + + /* control magic number */ + if (LizardF_readLE32(src) != LIZARDF_MAGICNUMBER) return (size_t)-LizardF_ERROR_frameType_unknown; + + /* Frame Header Size */ + { BYTE const FLG = ((const BYTE*)src)[4]; + U32 const contentSizeFlag = (FLG>>3) & _1BIT; + return contentSizeFlag ? maxFHSize : minFHSize; + } +} + + +/*! LizardF_decodeHeader() : + input : `srcVoidPtr` points at the **beginning of the frame** + output : set internal values of dctx, such as + dctxPtr->frameInfo and dctxPtr->dStage. + Also allocates internal buffers. + @return : nb Bytes read from srcVoidPtr (necessarily <= srcSize) + or an error code (testable with LizardF_isError()) +*/ +static size_t LizardF_decodeHeader(LizardF_dctx_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) +{ + BYTE FLG, BD, HC; + unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID; + size_t bufferNeeded, currentBlockSize; + size_t frameHeaderSize; + const BYTE* srcPtr = (const BYTE*)srcVoidPtr; + + /* need to decode header to get frameInfo */ + if (srcSize < minFHSize) return (size_t)-LizardF_ERROR_frameHeader_incomplete; /* minimal frame header size */ + memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo)); + + /* special case : skippable frames */ + if ((LizardF_readLE32(srcPtr) & 0xFFFFFFF0U) == LIZARDF_MAGIC_SKIPPABLE_START) { + dctxPtr->frameInfo.frameType = LizardF_skippableFrame; + if (srcVoidPtr == (void*)(dctxPtr->header)) { + dctxPtr->tmpInSize = srcSize; + dctxPtr->tmpInTarget = 8; + dctxPtr->dStage = dstage_storeSFrameSize; + return srcSize; + } else { + dctxPtr->dStage = dstage_getSFrameSize; + return 4; + } + } + + /* control magic number */ + if (LizardF_readLE32(srcPtr) != LIZARDF_MAGICNUMBER) return (size_t)-LizardF_ERROR_frameType_unknown; + dctxPtr->frameInfo.frameType = LizardF_frame; + + /* Flags */ + FLG = srcPtr[4]; + version = (FLG>>6) & _2BITS; + blockMode = (FLG>>5) & _1BIT; + blockChecksumFlag = (FLG>>4) & _1BIT; + contentSizeFlag = (FLG>>3) & _1BIT; + contentChecksumFlag = (FLG>>2) & _1BIT; + + /* Frame Header Size */ + frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize; + + if (srcSize < frameHeaderSize) { + /* not enough input to fully decode frame header */ + if (srcPtr != dctxPtr->header) + memcpy(dctxPtr->header, srcPtr, srcSize); + dctxPtr->tmpInSize = srcSize; + dctxPtr->tmpInTarget = frameHeaderSize; + dctxPtr->dStage = dstage_storeHeader; + return srcSize; + } + + BD = srcPtr[5]; + blockSizeID = (BD>>4) & _3BITS; + + /* validate */ + if (version != 1) return (size_t)-LizardF_ERROR_headerVersion_wrong; /* Version Number, only supported value */ + if (blockChecksumFlag != 0) return (size_t)-LizardF_ERROR_blockChecksum_unsupported; /* Not supported for the time being */ + if (((FLG>>0)&_2BITS) != 0) return (size_t)-LizardF_ERROR_reservedFlag_set; /* Reserved bits */ + if (((BD>>7)&_1BIT) != 0) return (size_t)-LizardF_ERROR_reservedFlag_set; /* Reserved bit */ + if (blockSizeID < 1) return (size_t)-LizardF_ERROR_maxBlockSize_invalid; /* 1-7 only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) return (size_t)-LizardF_ERROR_reservedFlag_set; /* Reserved bits */ + + /* check */ + HC = LizardF_headerChecksum(srcPtr+4, frameHeaderSize-5); + if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-LizardF_ERROR_headerChecksum_invalid; /* Bad header checksum error */ + + /* save */ + dctxPtr->frameInfo.blockMode = (LizardF_blockMode_t)blockMode; + dctxPtr->frameInfo.contentChecksumFlag = (LizardF_contentChecksum_t)contentChecksumFlag; + dctxPtr->frameInfo.blockSizeID = (LizardF_blockSizeID_t)blockSizeID; + currentBlockSize = dctxPtr->maxBlockSize; + dctxPtr->maxBlockSize = LizardF_getBlockSize(blockSizeID); + if (contentSizeFlag) + dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LizardF_readLE64(srcPtr+6); + + /* init */ + if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); + + /* alloc */ + bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LizardF_blockLinked) * 2 * LIZARD_DICT_SIZE); + if (bufferNeeded > dctxPtr->maxBufferSize || dctxPtr->maxBlockSize > currentBlockSize) { /* tmp buffers too small */ + FREEMEM(dctxPtr->tmpIn); + FREEMEM(dctxPtr->tmpOutBuffer); + dctxPtr->maxBufferSize = 0; + dctxPtr->tmpIn = (BYTE*)ALLOCATOR(1, dctxPtr->maxBlockSize); + if (dctxPtr->tmpIn == NULL) return (size_t)-LizardF_ERROR_GENERIC; + dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(1, bufferNeeded); + if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-LizardF_ERROR_GENERIC; + dctxPtr->maxBufferSize = bufferNeeded; + } + dctxPtr->tmpInSize = 0; + dctxPtr->tmpInTarget = 0; + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = 0; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; + dctxPtr->tmpOutStart = 0; + dctxPtr->tmpOutSize = 0; + + dctxPtr->dStage = dstage_getCBlockSize; + + return frameHeaderSize; +} + + +/*! LizardF_getFrameInfo() : +* Decodes frame header information, such as blockSize. +* It is optional : you could start by calling directly LizardF_decompress() instead. +* The objective is to extract header information without starting decompression, typically for allocation purposes. +* LizardF_getFrameInfo() can also be used *after* starting decompression, on a valid LizardF_decompressionContext_t. +* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). +* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) +* @return : hint of the better `srcSize` to use for next call to LizardF_decompress, +* or an error code which can be tested using LizardF_isError(). +*/ +LizardF_errorCode_t LizardF_getFrameInfo(LizardF_decompressionContext_t dCtx, LizardF_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr) +{ + LizardF_dctx_t* dctxPtr = (LizardF_dctx_t*)dCtx; + + if (dctxPtr->dStage > dstage_storeHeader) { /* note : requires dstage_* header related to be at beginning of enum */ + /* frameInfo already decoded */ + size_t o=0, i=0; + *srcSizePtr = 0; + *frameInfoPtr = dctxPtr->frameInfo; + return LizardF_decompress(dCtx, NULL, &o, NULL, &i, NULL); /* returns : recommended nb of bytes for LizardF_decompress() */ + } else { + size_t nextSrcSize, o=0; + size_t const hSize = LizardF_headerSize(srcBuffer, *srcSizePtr); + if (LizardF_isError(hSize)) { *srcSizePtr=0; return hSize; } + if (*srcSizePtr < hSize) { *srcSizePtr=0; return (size_t)-LizardF_ERROR_frameHeader_incomplete; } + + *srcSizePtr = hSize; + nextSrcSize = LizardF_decompress(dCtx, NULL, &o, srcBuffer, srcSizePtr, NULL); + if (dctxPtr->dStage <= dstage_storeHeader) return (size_t)-LizardF_ERROR_frameHeader_incomplete; /* should not happen, already checked */ + *frameInfoPtr = dctxPtr->frameInfo; + return nextSrcSize; + } +} + + +/* trivial redirector, for common prototype */ +static int LizardF_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) +{ + (void)dictStart; (void)dictSize; + return Lizard_decompress_safe (source, dest, compressedSize, maxDecompressedSize); +} + + +static void LizardF_updateDict(LizardF_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) +{ + if (dctxPtr->dictSize==0) + dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ + + if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) { /* dictionary continuity */ + dctxPtr->dictSize += dstSize; + return; + } + + if (dstPtr - dstPtr0 + dstSize >= LIZARD_DICT_SIZE) { /* dstBuffer large enough to become dictionary */ + dctxPtr->dict = (const BYTE*)dstPtr0; + dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize; + return; + } + + if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer)) { + /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */ + dctxPtr->dictSize += dstSize; + return; + } + + if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ + size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; + size_t copySize = LIZARD_DICT_SIZE - dctxPtr->tmpOutSize; + const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; + if (dctxPtr->tmpOutSize > LIZARD_DICT_SIZE) copySize = 0; + if (copySize > preserveSize) copySize = preserveSize; + + memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize; + return; + } + + if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { /* copy dst into tmp to complete dict */ + if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) { /* tmp buffer not large enough */ + size_t preserveSize = LIZARD_DICT_SIZE - dstSize; /* note : dstSize < LIZARD_DICT_SIZE */ + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); + dctxPtr->dictSize = preserveSize; + } + memcpy(dctxPtr->tmpOutBuffer + dctxPtr->dictSize, dstPtr, dstSize); + dctxPtr->dictSize += dstSize; + return; + } + + /* join dict & dest into tmp */ + { size_t preserveSize = LIZARD_DICT_SIZE - dstSize; /* note : dstSize < LIZARD_DICT_SIZE */ + if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize; + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); + memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize); + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = preserveSize + dstSize; + } +} + + + +/*! LizardF_decompress() : +* Call this function repetitively to regenerate data compressed within srcBuffer. +* The function will attempt to decode *srcSizePtr from srcBuffer, into dstBuffer of maximum size *dstSizePtr. +* +* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). +* +* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). +* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. +* You will have to call it again, continuing from where it stopped. +* +* The function result is an hint of the better srcSize to use for next call to LizardF_decompress. +* Basically, it's the size of the current (or remaining) compressed block + header of next block. +* Respecting the hint provides some boost to performance, since it allows less buffer shuffling. +* Note that this is just a hint, you can always provide any srcSize you want. +* When a frame is fully decoded, the function result will be 0. +* If decompression failed, function result is an error code which can be tested using LizardF_isError(). +*/ +size_t LizardF_decompress(LizardF_decompressionContext_t decompressionContext, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LizardF_decompressOptions_t* decompressOptionsPtr) +{ + LizardF_dctx_t* dctxPtr = (LizardF_dctx_t*)decompressionContext; + LizardF_decompressOptions_t optionsNull; + const BYTE* const srcStart = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcStart + *srcSizePtr; + const BYTE* srcPtr = srcStart; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* const dstEnd = dstStart + *dstSizePtr; + BYTE* dstPtr = dstStart; + const BYTE* selectedIn = NULL; + unsigned doAnotherStage = 1; + size_t nextSrcSizeHint = 1; + + + memset(&optionsNull, 0, sizeof(optionsNull)); + if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; + *srcSizePtr = 0; + *dstSizePtr = 0; + + /* expect to continue decoding src buffer where it left previously */ + if (dctxPtr->srcExpect != NULL) { + if (srcStart != dctxPtr->srcExpect) return (size_t)-LizardF_ERROR_srcPtr_wrong; + } + + /* programmed as a state machine */ + + while (doAnotherStage) { + + switch(dctxPtr->dStage) + { + + case dstage_getHeader: + if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */ + LizardF_errorCode_t const hSize = LizardF_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr); + if (LizardF_isError(hSize)) return hSize; + srcPtr += hSize; + break; + } + dctxPtr->tmpInSize = 0; + dctxPtr->tmpInTarget = minFHSize; /* minimum to attempt decode */ + dctxPtr->dStage = dstage_storeHeader; + /* fallthrough */ + + case dstage_storeHeader: + { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + dctxPtr->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { + nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ + doAnotherStage = 0; /* not enough src data, ask for some more */ + break; + } + { LizardF_errorCode_t const hSize = LizardF_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); + if (LizardF_isError(hSize)) return hSize; + } + break; + } + + case dstage_getCBlockSize: + if ((size_t)(srcEnd - srcPtr) >= BHSize) { + selectedIn = srcPtr; + srcPtr += BHSize; + } else { + /* not enough input to read cBlockSize field */ + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeCBlockSize; + } + + if (dctxPtr->dStage == dstage_storeCBlockSize) /* can be skipped */ + case dstage_storeCBlockSize: + { + size_t sizeToCopy = BHSize - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < BHSize) { /* not enough input to get full cBlockSize; wait for more */ + nextSrcSizeHint = BHSize - dctxPtr->tmpInSize; + doAnotherStage = 0; + break; + } + selectedIn = dctxPtr->tmpIn; + } + + /* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */ + { size_t const nextCBlockSize = LizardF_readLE32(selectedIn) & 0x7FFFFFFFU; + if (nextCBlockSize==0) { /* frameEnd signal, no more CBlock */ + dctxPtr->dStage = dstage_getSuffix; + break; + } + if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-LizardF_ERROR_GENERIC; /* invalid cBlockSize */ + dctxPtr->tmpInTarget = nextCBlockSize; + if (LizardF_readLE32(selectedIn) & LIZARDF_BLOCKUNCOMPRESSED_FLAG) { + dctxPtr->dStage = dstage_copyDirect; + break; + } + dctxPtr->dStage = dstage_getCBlock; + if (dstPtr==dstEnd) { + nextSrcSizeHint = nextCBlockSize + BHSize; + doAnotherStage = 0; + } + break; + } + + case dstage_copyDirect: /* uncompressed block */ + { size_t sizeToCopy = dctxPtr->tmpInTarget; + if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */ + if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; + memcpy(dstPtr, srcPtr, sizeToCopy); + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy); + if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy; + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==LizardF_blockLinked) + LizardF_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); + + srcPtr += sizeToCopy; + dstPtr += sizeToCopy; + if (sizeToCopy == dctxPtr->tmpInTarget) { /* all copied */ + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */ + nextSrcSizeHint = dctxPtr->tmpInTarget + BHSize; + doAnotherStage = 0; + break; + } + + case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */ + if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) { + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeCBlock; + break; + } + selectedIn = srcPtr; + srcPtr += dctxPtr->tmpInTarget; + dctxPtr->dStage = dstage_decodeCBlock; + break; + + case dstage_storeCBlock: + { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + dctxPtr->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* need more input */ + nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; + doAnotherStage=0; + break; + } + selectedIn = dctxPtr->tmpIn; + dctxPtr->dStage = dstage_decodeCBlock; + } + /* fallthrough */ + + case dstage_decodeCBlock: + if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */ + dctxPtr->dStage = dstage_decodeCBlock_intoTmp; + else + dctxPtr->dStage = dstage_decodeCBlock_intoDst; + break; + + case dstage_decodeCBlock_intoDst: + { int (*decoder)(const char*, char*, int, int, const char*, int); + int decodedSize; + + if (dctxPtr->frameInfo.blockMode == LizardF_blockLinked) + decoder = Lizard_decompress_safe_usingDict; + else + decoder = LizardF_decompress_safe; + + decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + if (decodedSize < 0) return (size_t)-LizardF_ERROR_GENERIC; /* decompression failed */ + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); + if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==LizardF_blockLinked) + LizardF_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); + + dstPtr += decodedSize; + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + + case dstage_decodeCBlock_intoTmp: + /* not enough place into dst : decode into tmpOut */ + { int (*decoder)(const char*, char*, int, int, const char*, int); + int decodedSize; + + if (dctxPtr->frameInfo.blockMode == LizardF_blockLinked) + decoder = Lizard_decompress_safe_usingDict; + else + decoder = LizardF_decompress_safe; + + /* ensure enough place for tmpOut */ + if (dctxPtr->frameInfo.blockMode == LizardF_blockLinked) { + if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { + if (dctxPtr->dictSize > 2 * LIZARD_DICT_SIZE) { + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - LIZARD_DICT_SIZE, LIZARD_DICT_SIZE); + dctxPtr->dictSize = LIZARD_DICT_SIZE; + } + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize; + } else { /* dict not within tmp */ + size_t reservedDictSpace = dctxPtr->dictSize; + if (reservedDictSpace > LIZARD_DICT_SIZE) reservedDictSpace = LIZARD_DICT_SIZE; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace; + } + } + + /* Decode */ + decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + if (decodedSize < 0) return (size_t)-LizardF_ERROR_decompressionFailed; /* decompression failed */ + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); + if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; + dctxPtr->tmpOutSize = decodedSize; + dctxPtr->tmpOutStart = 0; + dctxPtr->dStage = dstage_flushOut; + break; + } + + case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ + { size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart; + if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr; + memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==LizardF_blockLinked) + LizardF_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); + + dctxPtr->tmpOutStart += sizeToCopy; + dstPtr += sizeToCopy; + + /* end of flush ? */ + if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) { + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + nextSrcSizeHint = BHSize; + doAnotherStage = 0; /* still some data to flush */ + break; + } + + case dstage_getSuffix: + { size_t const suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; + if (dctxPtr->frameRemainingSize) return (size_t)-LizardF_ERROR_frameSize_wrong; /* incorrect frame size decoded */ + if (suffixSize == 0) { /* frame completed */ + nextSrcSizeHint = 0; + dctxPtr->dStage = dstage_getHeader; + doAnotherStage = 0; + break; + } + if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */ + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeSuffix; + } else { + selectedIn = srcPtr; + srcPtr += 4; + } + } + + if (dctxPtr->dStage == dstage_storeSuffix) /* can be skipped */ + case dstage_storeSuffix: + { + size_t sizeToCopy = 4 - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < 4) { /* not enough input to read complete suffix */ + nextSrcSizeHint = 4 - dctxPtr->tmpInSize; + doAnotherStage=0; + break; + } + selectedIn = dctxPtr->tmpIn; + } + + /* case dstage_checkSuffix: */ /* no direct call, to avoid scan-build warning */ + { U32 const readCRC = LizardF_readLE32(selectedIn); + U32 const resultCRC = XXH32_digest(&(dctxPtr->xxh)); + if (readCRC != resultCRC) return (size_t)-LizardF_ERROR_contentChecksum_invalid; + nextSrcSizeHint = 0; + dctxPtr->dStage = dstage_getHeader; + doAnotherStage = 0; + break; + } + + case dstage_getSFrameSize: + if ((srcEnd - srcPtr) >= 4) { + selectedIn = srcPtr; + srcPtr += 4; + } else { + /* not enough input to read cBlockSize field */ + dctxPtr->tmpInSize = 4; + dctxPtr->tmpInTarget = 8; + dctxPtr->dStage = dstage_storeSFrameSize; + } + + if (dctxPtr->dStage == dstage_storeSFrameSize) + case dstage_storeSFrameSize: + { + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* not enough input to get full sBlockSize; wait for more */ + nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + doAnotherStage = 0; + break; + } + selectedIn = dctxPtr->header + 4; + } + + /* case dstage_decodeSFrameSize: */ /* no direct access */ + { size_t const SFrameSize = LizardF_readLE32(selectedIn); + dctxPtr->frameInfo.contentSize = SFrameSize; + dctxPtr->tmpInTarget = SFrameSize; + dctxPtr->dStage = dstage_skipSkippable; + break; + } + + case dstage_skipSkippable: + { size_t skipSize = dctxPtr->tmpInTarget; + if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr; + srcPtr += skipSize; + dctxPtr->tmpInTarget -= skipSize; + doAnotherStage = 0; + nextSrcSizeHint = dctxPtr->tmpInTarget; + if (nextSrcSizeHint) break; + dctxPtr->dStage = dstage_getHeader; + break; + } + } + } + + /* preserve dictionary within tmp if necessary */ + if ( (dctxPtr->frameInfo.blockMode==LizardF_blockLinked) + &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) + &&(!decompressOptionsPtr->stableDst) + &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) + ) + { + if (dctxPtr->dStage == dstage_flushOut) { + size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; + size_t copySize = LIZARD_DICT_SIZE - dctxPtr->tmpOutSize; + const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; + if (dctxPtr->tmpOutSize > LIZARD_DICT_SIZE) copySize = 0; + if (copySize > preserveSize) copySize = preserveSize; + + memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart; + } else { + size_t newDictSize = dctxPtr->dictSize; + const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; + if ((newDictSize) > LIZARD_DICT_SIZE) newDictSize = LIZARD_DICT_SIZE; + + memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); + + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = newDictSize; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize; + } + } + + /* require function to be called again from position where it stopped */ + if (srcPtrsrcExpect = srcPtr; + else + dctxPtr->srcExpect = NULL; + + *srcSizePtr = (srcPtr - srcStart); + *dstSizePtr = (dstPtr - dstStart); + return nextSrcSizeHint; +} diff --git a/contrib/lizard/lib/lizard_frame.h b/contrib/lizard/lib/lizard_frame.h new file mode 100644 index 00000000000..7cdd20063c5 --- /dev/null +++ b/contrib/lizard/lib/lizard_frame.h @@ -0,0 +1,303 @@ +/* + Lizard auto-framing library + Header File + Copyright (C) 2011-2015, Yann Collet + Copyright (C) 2016-2017, Przemyslaw Skibinski + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + +/* LizardF is a stand-alone API to create Lizard-compressed frames + * conformant with specification v1.5.1. + * All related operations, including memory management, are handled internally by the library. + * You don't need lizard_compress.h when using lizard_frame.h. + * */ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************ +* Includes +**************************************/ +#include /* size_t */ + + +/*-************************************ +* Error management +**************************************/ +typedef size_t LizardF_errorCode_t; + +unsigned LizardF_isError(LizardF_errorCode_t code); +const char* LizardF_getErrorName(LizardF_errorCode_t code); /* return error code string; useful for debugging */ + + +/*-************************************ +* Frame compression types +**************************************/ +//#define LIZARDF_DISABLE_OBSOLETE_ENUMS +#ifndef LIZARDF_DISABLE_OBSOLETE_ENUMS +# define LIZARDF_OBSOLETE_ENUM(x) ,x +#else +# define LIZARDF_OBSOLETE_ENUM(x) +#endif + +typedef enum { + LizardF_default=0, + LizardF_max128KB=1, + LizardF_max256KB=2, + LizardF_max1MB=3, + LizardF_max4MB=4, + LizardF_max16MB=5, + LizardF_max64MB=6, + LizardF_max256MB=7 +} LizardF_blockSizeID_t; + +typedef enum { + LizardF_blockLinked=0, + LizardF_blockIndependent + LIZARDF_OBSOLETE_ENUM(blockLinked = LizardF_blockLinked) + LIZARDF_OBSOLETE_ENUM(blockIndependent = LizardF_blockIndependent) +} LizardF_blockMode_t; + +typedef enum { + LizardF_noContentChecksum=0, + LizardF_contentChecksumEnabled + LIZARDF_OBSOLETE_ENUM(noContentChecksum = LizardF_noContentChecksum) + LIZARDF_OBSOLETE_ENUM(contentChecksumEnabled = LizardF_contentChecksumEnabled) +} LizardF_contentChecksum_t; + +typedef enum { + LizardF_frame=0, + LizardF_skippableFrame + LIZARDF_OBSOLETE_ENUM(skippableFrame = LizardF_skippableFrame) +} LizardF_frameType_t; + +#ifndef LIZARDF_DISABLE_OBSOLETE_ENUMS +typedef LizardF_blockSizeID_t blockSizeID_t; +typedef LizardF_blockMode_t blockMode_t; +typedef LizardF_frameType_t frameType_t; +typedef LizardF_contentChecksum_t contentChecksum_t; +#endif + +typedef struct { + LizardF_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + LizardF_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ + LizardF_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ + LizardF_frameType_t frameType; /* LizardF_frame, skippableFrame ; 0 == default */ + unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */ + unsigned reserved[2]; /* must be zero for forward compatibility */ +} LizardF_frameInfo_t; + +typedef struct { + LizardF_frameInfo_t frameInfo; + int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */ + unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */ + unsigned reserved[4]; /* must be zero for forward compatibility */ +} LizardF_preferences_t; + + +/*-********************************* +* Simple compression function +***********************************/ +size_t LizardF_compressFrameBound(size_t srcSize, const LizardF_preferences_t* preferencesPtr); + +/*!LizardF_compressFrame() : + * Compress an entire srcBuffer into a valid Lizard frame, as defined by specification v1.5.1 + * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. + * You can get the minimum value of dstMaxSize by using LizardF_compressFrameBound() + * If this condition is not respected, LizardF_compressFrame() will fail (result is an errorCode) + * The LizardF_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + * The result of the function is the number of bytes written into dstBuffer. + * The function outputs an error code if it fails (can be tested using LizardF_isError()) + */ +size_t LizardF_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_preferences_t* preferencesPtr); + + + +/*-*********************************** +* Advanced compression functions +*************************************/ +typedef struct LizardF_cctx_s* LizardF_compressionContext_t; /* must be aligned on 8-bytes */ + +typedef struct { + unsigned stableSrc; /* 1 == src content will remain available on future calls to LizardF_compress(); avoid saving src content within tmp buffer as future dictionary */ + unsigned reserved[3]; +} LizardF_compressOptions_t; + +/* Resource Management */ + +#define LIZARDF_VERSION 100 +LizardF_errorCode_t LizardF_createCompressionContext(LizardF_compressionContext_t* cctxPtr, unsigned version); +LizardF_errorCode_t LizardF_freeCompressionContext(LizardF_compressionContext_t cctx); +/* LizardF_createCompressionContext() : + * The first thing to do is to create a compressionContext object, which will be used in all compression operations. + * This is achieved using LizardF_createCompressionContext(), which takes as argument a version and an LizardF_preferences_t structure. + * The version provided MUST be LIZARDF_VERSION. It is intended to track potential version differences between different binaries. + * The function will provide a pointer to a fully allocated LizardF_compressionContext_t object. + * If the result LizardF_errorCode_t is not zero, there was an error during context creation. + * Object can release its memory using LizardF_freeCompressionContext(); + */ + + +/* Compression */ + +size_t LizardF_compressBegin(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LizardF_preferences_t* prefsPtr); +/* LizardF_compressBegin() : + * will write the frame header into dstBuffer. + * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes. + * The LizardF_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. + * The result of the function is the number of bytes written into dstBuffer for the header + * or an error code (can be tested using LizardF_isError()) + */ + +size_t LizardF_compressBound(size_t srcSize, const LizardF_preferences_t* prefsPtr); +/* LizardF_compressBound() : + * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. + * Different preferences can produce different results. + * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. + * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) + */ + +size_t LizardF_compressUpdate(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_compressOptions_t* cOptPtr); +/* LizardF_compressUpdate() + * LizardF_compressUpdate() can be called repetitively to compress as much data as necessary. + * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. + * You can get the minimum value of dstMaxSize by using LizardF_compressBound(). + * If this condition is not respected, LizardF_compress() will fail (result is an errorCode). + * LizardF_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs. + * The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. + * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. + * The function outputs an error code if it fails (can be tested using LizardF_isError()) + */ + +size_t LizardF_flush(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* cOptPtr); +/* LizardF_flush() + * Should you need to generate compressed data immediately, without waiting for the current block to be filled, + * you can call Lizard_flush(), which will immediately compress any remaining data buffered within cctx. + * Note that dstMaxSize must be large enough to ensure the operation will be successful. + * LizardF_compressOptions_t structure is optional : you can provide NULL as argument. + * The result of the function is the number of bytes written into dstBuffer + * (it can be zero, this means there was no data left within cctx) + * The function outputs an error code if it fails (can be tested using LizardF_isError()) + */ + +size_t LizardF_compressEnd(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* cOptPtr); +/* LizardF_compressEnd() + * When you want to properly finish the compressed frame, just call LizardF_compressEnd(). + * It will flush whatever data remained within compressionContext (like Lizard_flush()) + * but also properly finalize the frame, with an endMark and a checksum. + * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) + * The function outputs an error code if it fails (can be tested using LizardF_isError()) + * The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. + * A successful call to LizardF_compressEnd() makes cctx available again for next compression task. + */ + + +/*-********************************* +* Decompression functions +***********************************/ + +typedef struct LizardF_dctx_s* LizardF_decompressionContext_t; /* must be aligned on 8-bytes */ + +typedef struct { + unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */ + unsigned reserved[3]; +} LizardF_decompressOptions_t; + + +/* Resource management */ + +/*!LizardF_createDecompressionContext() : + * Create an LizardF_decompressionContext_t object, which will be used to track all decompression operations. + * The version provided MUST be LIZARDF_VERSION. It is intended to track potential breaking differences between different versions. + * The function will provide a pointer to a fully allocated and initialized LizardF_decompressionContext_t object. + * The result is an errorCode, which can be tested using LizardF_isError(). + * dctx memory can be released using LizardF_freeDecompressionContext(); + * The result of LizardF_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. + * That is, it should be == 0 if decompression has been completed fully and correctly. + */ +LizardF_errorCode_t LizardF_createDecompressionContext(LizardF_decompressionContext_t* dctxPtr, unsigned version); +LizardF_errorCode_t LizardF_freeDecompressionContext(LizardF_decompressionContext_t dctx); + + +/*====== Decompression ======*/ + +/*!LizardF_getFrameInfo() : + * This function decodes frame header information (such as max blockSize, frame checksum, etc.). + * Its usage is optional. The objective is to extract frame header information, typically for allocation purposes. + * A header size is variable and can be from 7 to 15 bytes. It's also possible to input more bytes than that. + * The number of bytes read from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). + * (note that LizardF_getFrameInfo() can also be used anytime *after* starting decompression, in this case 0 input byte is enough) + * Frame header info is *copied into* an already allocated LizardF_frameInfo_t structure. + * The function result is an hint about how many srcSize bytes LizardF_decompress() expects for next call, + * or an error code which can be tested using LizardF_isError() + * (typically, when there is not enough src bytes to fully decode the frame header) + * Decompression is expected to resume from where it stopped (srcBuffer + *srcSizePtr) + */ +size_t LizardF_getFrameInfo(LizardF_decompressionContext_t dctx, + LizardF_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr); + +/*!LizardF_decompress() : + * Call this function repetitively to regenerate data compressed within srcBuffer. + * The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr. + * + * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). + * + * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). + * If number of bytes read is < number of bytes provided, then decompression operation is not completed. + * It typically happens when dstBuffer is not large enough to contain all decoded data. + * LizardF_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr) + * The function will check this condition, and refuse to continue if it is not respected. + * + * `dstBuffer` is expected to be flushed between each call to the function, its content will be overwritten. + * `dst` arguments can be changed at will at each consecutive call to the function. + * + * The function result is an hint of how many `srcSize` bytes LizardF_decompress() expects for next call. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides some boost to performance, since it does skip intermediate buffers. + * This is just a hint though, it's always possible to provide any srcSize. + * When a frame is fully decoded, the function result will be 0 (no more data expected). + * If decompression failed, function result is an error code, which can be tested using LizardF_isError(). + * + * After a frame is fully decoded, dctx can be used again to decompress another frame. + */ +size_t LizardF_decompress(LizardF_decompressionContext_t dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LizardF_decompressOptions_t* dOptPtr); + + + +#if defined (__cplusplus) +} +#endif diff --git a/contrib/lizard/lib/lizard_frame_static.h b/contrib/lizard/lib/lizard_frame_static.h new file mode 100644 index 00000000000..da80756057b --- /dev/null +++ b/contrib/lizard/lib/lizard_frame_static.h @@ -0,0 +1,81 @@ +/* + Lizard auto-framing library + Header File for static linking only + Copyright (C) 2011-2015, Yann Collet. + Copyright (C) 2016-2017, Przemyslaw Skibinski + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + +/* lizard_frame_static.h should be used solely in the context of static linking. + * It contains definitions which may still change overtime. + * Never use it in the context of DLL linking. + * */ + + +/************************************** +* Includes +**************************************/ +#include "lizard_frame.h" + + +/************************************** + * Error management + * ************************************/ +#define LIZARDF_LIST_ERRORS(ITEM) \ + ITEM(OK_NoError) ITEM(ERROR_GENERIC) \ + ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \ + ITEM(ERROR_compressionLevel_invalid) \ + ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \ + ITEM(ERROR_allocation_failed) \ + ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \ + ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \ + ITEM(ERROR_srcPtr_wrong) \ + ITEM(ERROR_decompressionFailed) \ + ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \ + ITEM(ERROR_maxCode) + +//#define LIZARDF_DISABLE_OLD_ENUMS +#ifndef LIZARDF_DISABLE_OLD_ENUMS +#define LIZARDF_GENERATE_ENUM(ENUM) LizardF_##ENUM, ENUM = LizardF_##ENUM, +#else +#define LIZARDF_GENERATE_ENUM(ENUM) LizardF_##ENUM, +#endif +typedef enum { LIZARDF_LIST_ERRORS(LIZARDF_GENERATE_ENUM) } LizardF_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */ + + +#if defined (__cplusplus) +} +#endif diff --git a/contrib/lizard/lib/lizard_parser_fast.h b/contrib/lizard/lib/lizard_parser_fast.h new file mode 100644 index 00000000000..f9e54b7f475 --- /dev/null +++ b/contrib/lizard/lib/lizard_parser_fast.h @@ -0,0 +1,196 @@ +#define LIZARD_FAST_MIN_OFFSET 8 +#define LIZARD_FAST_LONGOFF_MM 0 /* not used with offsets > 1<<16 */ + +/************************************** +* Hash Functions +**************************************/ +static size_t Lizard_hashPosition(const void* p) +{ + if (MEM_64bits()) + return Lizard_hash5Ptr(p, LIZARD_HASHLOG_LZ4); + return Lizard_hash4Ptr(p, LIZARD_HASHLOG_LZ4); +} + +static void Lizard_putPositionOnHash(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase) +{ + hashTable[h] = (U32)(p-srcBase); +} + +static void Lizard_putPosition(const BYTE* p, U32* hashTable, const BYTE* srcBase) +{ + size_t const h = Lizard_hashPosition(p); + Lizard_putPositionOnHash(p, h, hashTable, srcBase); +} + +static U32 Lizard_getPositionOnHash(size_t h, U32* hashTable) +{ + return hashTable[h]; +} + +static U32 Lizard_getPosition(const BYTE* p, U32* hashTable) +{ + size_t const h = Lizard_hashPosition(p); + return Lizard_getPositionOnHash(h, hashTable); +} + + +static const U32 Lizard_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */ +static const U32 Lizard_minLength = (MFLIMIT+1); + + +FORCE_INLINE int Lizard_compress_fast( + Lizard_stream_t* const ctx, + const BYTE* ip, + const BYTE* const iend) +{ + const U32 acceleration = 1; + const BYTE* base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + const BYTE* anchor = ip; + + size_t forwardH, matchIndex; + const U32 maxDistance = (1 << ctx->params.windowLog) - 1; + const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip - base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; + + /* Init conditions */ + if ((U32)(iend-ip) > (U32)LIZARD_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */ + + if ((U32)(iend-ip) < Lizard_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + + /* First Byte */ + Lizard_putPosition(ip, ctx->hashTable, base); + ip++; forwardH = Lizard_hashPosition(ip); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + size_t matchLength; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << Lizard_skipTrigger; + while (1) { + size_t const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> Lizard_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + matchIndex = Lizard_getPositionOnHash(h, ctx->hashTable); + forwardH = Lizard_hashPosition(forwardIp); + Lizard_putPositionOnHash(ip, h, ctx->hashTable, base); + + if ((matchIndex < lowLimit) || (matchIndex >= (U32)(ip - base)) || (base + matchIndex + maxDistance < ip)) continue; + + if (matchIndex >= dictLimit) { + match = base + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) +#endif + if (MEM_read32(match) == MEM_read32(ip)) + { + int back = 0; + matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); + + while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; + matchLength -= back; +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + { + ip += back; + match += back; + break; + } + } + } else { + match = dictBase + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) +#endif + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance; + int back = 0; + matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); + + while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--; + matchLength -= back; + match = base + matchIndex + back; +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + { + ip += back; + break; + } + } + } + } // while (1) + } + +_next_match: + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + Lizard_putPosition(ip-2, ctx->hashTable, base); + + /* Test next position */ + matchIndex = Lizard_getPosition(ip, ctx->hashTable); + Lizard_putPosition(ip, ctx->hashTable, base); + if ((matchIndex >= lowLimit) && (matchIndex < (U32)(ip - base)) && (base + matchIndex + maxDistance >= ip)) + { + if (matchIndex >= dictLimit) { + match = base + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) +#endif + if (MEM_read32(match) == MEM_read32(ip)) + { + matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + goto _next_match; + } + } else { + match = dictBase + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) +#endif + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); + match = base + matchIndex; +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + goto _next_match; + } + } + } + + /* Prepare next loop */ + forwardH = Lizard_hashPosition(++ip); + } + +_last_literals: + /* Encode Last Literals */ + ip = iend; + if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; + + /* End */ + return 1; +_output_error: + return 0; +} diff --git a/contrib/lizard/lib/lizard_parser_fastbig.h b/contrib/lizard/lib/lizard_parser_fastbig.h new file mode 100644 index 00000000000..30c464fcb90 --- /dev/null +++ b/contrib/lizard/lib/lizard_parser_fastbig.h @@ -0,0 +1,175 @@ +#define LIZARD_FASTBIG_LONGOFF_MM MM_LONGOFF + +/************************************** +* Hash Functions +**************************************/ +static size_t Lizard_hashPositionHLog(const void* p, int hashLog) +{ + if (MEM_64bits()) + return Lizard_hash5Ptr(p, hashLog); + return Lizard_hash4Ptr(p, hashLog); +} + +static void Lizard_putPositionOnHashHLog(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase) +{ + hashTable[h] = (U32)(p-srcBase); +} + +static void Lizard_putPositionHLog(const BYTE* p, U32* hashTable, const BYTE* srcBase, int hashLog) +{ + size_t const h = Lizard_hashPositionHLog(p, hashLog); + Lizard_putPositionOnHashHLog(p, h, hashTable, srcBase); +} + +static U32 Lizard_getPositionOnHashHLog(size_t h, U32* hashTable) +{ + return hashTable[h]; +} + +static U32 Lizard_getPositionHLog(const BYTE* p, U32* hashTable, int hashLog) +{ + size_t const h = Lizard_hashPositionHLog(p, hashLog); + return Lizard_getPositionOnHashHLog(h, hashTable); +} + +FORCE_INLINE int Lizard_compress_fastBig( + Lizard_stream_t* const ctx, + const BYTE* ip, + const BYTE* const iend) +{ + const U32 acceleration = 1; + const BYTE* base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + const BYTE* anchor = ip; + + size_t forwardH, matchIndex; + const int hashLog = ctx->params.hashLog; + const U32 maxDistance = (1 << ctx->params.windowLog) - 1; + const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip - base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; + + /* Init conditions */ + if ((U32)(iend-ip) > (U32)LIZARD_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */ + + if ((U32)(iend-ip) < Lizard_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + + /* First Byte */ + Lizard_putPositionHLog(ip, ctx->hashTable, base, hashLog); + ip++; forwardH = Lizard_hashPositionHLog(ip, hashLog); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + size_t matchLength; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << Lizard_skipTrigger; + while (1) { + size_t const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> Lizard_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + matchIndex = Lizard_getPositionOnHashHLog(h, ctx->hashTable); + forwardH = Lizard_hashPositionHLog(forwardIp, hashLog); + Lizard_putPositionOnHashHLog(ip, h, ctx->hashTable, base); + + if ((matchIndex < lowLimit) || (matchIndex >= (U32)(ip - base)) || (base + matchIndex + maxDistance < ip)) continue; + + if (matchIndex >= dictLimit) { + match = base + matchIndex; + if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) + if (MEM_read32(match) == MEM_read32(ip)) + { + int back = 0; + matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); + + while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; + matchLength -= back; + if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + { + ip += back; + match += back; + break; + } + } + } else { + match = dictBase + matchIndex; + if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance; + int back = 0; + matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); + + while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--; + matchLength -= back; + match = base + matchIndex + back; + if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + { + ip += back; + break; + } + } + } + } // while (1) + } + +_next_match: + if (Lizard_encodeSequence_LIZv1(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + Lizard_putPositionHLog(ip-2, ctx->hashTable, base, hashLog); + + /* Test next position */ + matchIndex = Lizard_getPositionHLog(ip, ctx->hashTable, hashLog); + Lizard_putPositionHLog(ip, ctx->hashTable, base, hashLog); + if ((matchIndex >= lowLimit) && (matchIndex < (U32)(ip - base)) && (base + matchIndex + maxDistance >= ip)) + { + if (matchIndex >= dictLimit) { + match = base + matchIndex; + if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) + if (MEM_read32(match) == MEM_read32(ip)) + { + matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); + if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + goto _next_match; + } + } else { + match = dictBase + matchIndex; + if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); + match = base + matchIndex; + if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + goto _next_match; + } + } + } + + /* Prepare next loop */ + forwardH = Lizard_hashPositionHLog(++ip, hashLog); + } + +_last_literals: + /* Encode Last Literals */ + ip = iend; + if (Lizard_encodeLastLiterals_LIZv1(ctx, &ip, &anchor)) goto _output_error; + + /* End */ + return 1; +_output_error: + return 0; +} diff --git a/contrib/lizard/lib/lizard_parser_fastsmall.h b/contrib/lizard/lib/lizard_parser_fastsmall.h new file mode 100644 index 00000000000..550edc5b201 --- /dev/null +++ b/contrib/lizard/lib/lizard_parser_fastsmall.h @@ -0,0 +1,189 @@ +/************************************** +* Hash Functions +**************************************/ +static size_t Lizard_hashPositionSmall(const void* p) +{ + if (MEM_64bits()) + return Lizard_hash5Ptr(p, LIZARD_HASHLOG_LZ4SM); + return Lizard_hash4Ptr(p, LIZARD_HASHLOG_LZ4SM); +} + +static void Lizard_putPositionOnHashSmall(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase) +{ + hashTable[h] = (U32)(p-srcBase); +} + +static void Lizard_putPositionSmall(const BYTE* p, U32* hashTable, const BYTE* srcBase) +{ + size_t const h = Lizard_hashPositionSmall(p); + Lizard_putPositionOnHashSmall(p, h, hashTable, srcBase); +} + +static U32 Lizard_getPositionOnHashSmall(size_t h, U32* hashTable) +{ + return hashTable[h]; +} + +static U32 Lizard_getPositionSmall(const BYTE* p, U32* hashTable) +{ + size_t const h = Lizard_hashPositionSmall(p); + return Lizard_getPositionOnHashSmall(h, hashTable); +} + + +FORCE_INLINE int Lizard_compress_fastSmall( + Lizard_stream_t* const ctx, + const BYTE* ip, + const BYTE* const iend) +{ + const U32 acceleration = 1; + const BYTE* base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + const BYTE* anchor = ip; + + size_t forwardH, matchIndex; + const U32 maxDistance = (1 << ctx->params.windowLog) - 1; + const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip - base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; + + /* Init conditions */ + if ((U32)(iend-ip) > (U32)LIZARD_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */ + + if ((U32)(iend-ip) < Lizard_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + + /* First Byte */ + Lizard_putPositionSmall(ip, ctx->hashTable, base); + ip++; forwardH = Lizard_hashPositionSmall(ip); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + size_t matchLength; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << Lizard_skipTrigger; + while (1) { + size_t const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> Lizard_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + matchIndex = Lizard_getPositionOnHashSmall(h, ctx->hashTable); + forwardH = Lizard_hashPositionSmall(forwardIp); + Lizard_putPositionOnHashSmall(ip, h, ctx->hashTable, base); + + if ((matchIndex < lowLimit) || (matchIndex >= (U32)(ip - base)) || (base + matchIndex + maxDistance < ip)) continue; + + if (matchIndex >= dictLimit) { + match = base + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) +#endif + if (MEM_read32(match) == MEM_read32(ip)) + { + int back = 0; + matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); + + while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; + matchLength -= back; +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + { + ip += back; + match += back; + break; + } + } + } else { + match = dictBase + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) +#endif + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance; + int back = 0; + matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); + + while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--; + matchLength -= back; + match = base + matchIndex + back; +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + { + ip += back; + break; + } + } + } + } // while (1) + } + +_next_match: + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + Lizard_putPositionSmall(ip-2, ctx->hashTable, base); + + /* Test next position */ + matchIndex = Lizard_getPositionSmall(ip, ctx->hashTable); + Lizard_putPositionSmall(ip, ctx->hashTable, base); + if ((matchIndex >= lowLimit) && (matchIndex < (U32)(ip - base)) && (base + matchIndex + maxDistance >= ip)) + { + if (matchIndex >= dictLimit) { + match = base + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) +#endif + if (MEM_read32(match) == MEM_read32(ip)) + { + matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + goto _next_match; + } + } else { + match = dictBase + matchIndex; +#if LIZARD_FAST_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) +#endif + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); + match = base + matchIndex; +#if LIZARD_FAST_LONGOFF_MM > 0 + if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + goto _next_match; + } + } + } + + /* Prepare next loop */ + forwardH = Lizard_hashPositionSmall(++ip); + } + +_last_literals: + /* Encode Last Literals */ + ip = iend; + if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; + + /* End */ + return 1; +_output_error: + return 0; +} diff --git a/contrib/lizard/lib/lizard_parser_hashchain.h b/contrib/lizard/lib/lizard_parser_hashchain.h new file mode 100644 index 00000000000..64b9d76c451 --- /dev/null +++ b/contrib/lizard/lib/lizard_parser_hashchain.h @@ -0,0 +1,369 @@ +#define LIZARD_HC_MIN_OFFSET 8 +#define LIZARD_HC_LONGOFF_MM 0 /* not used with offsets > 1<<16 */ +#define OPTIMAL_ML (int)((ML_MASK_LZ4-1)+MINMATCH) +#define GET_MINMATCH(offset) (MINMATCH) + +#if 1 + #define LIZARD_HC_HASH_FUNCTION(ip, hashLog) Lizard_hashPtr(ip, hashLog, ctx->params.searchLength) +#else + #define LIZARD_HC_HASH_FUNCTION(ip, hashLog) Lizard_hash5Ptr(ip, hashLog) +#endif + +/* Update chains up to ip (excluded) */ +FORCE_INLINE void Lizard_Insert (Lizard_stream_t* ctx, const BYTE* ip) +{ + U32* const chainTable = ctx->chainTable; + U32* const hashTable = ctx->hashTable; +#if MINMATCH == 3 + U32* HashTable3 = ctx->hashTable3; +#endif + const BYTE* const base = ctx->base; + U32 const target = (U32)(ip - base); + U32 idx = ctx->nextToUpdate; + const int hashLog = ctx->params.hashLog; + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + const U32 maxDistance = (1 << ctx->params.windowLog) - 1; + + while (idx < target) { + size_t const h = Lizard_hashPtr(base+idx, hashLog, ctx->params.searchLength); + size_t delta = idx - hashTable[h]; + if (delta>maxDistance) delta = maxDistance; + DELTANEXT(idx) = (U32)delta; + if ((hashTable[h] >= idx) || (idx >= hashTable[h] + LIZARD_HC_MIN_OFFSET)) + hashTable[h] = idx; +#if MINMATCH == 3 + HashTable3[Lizard_hash3Ptr(base+idx, ctx->params.hashLog3)] = idx; +#endif + idx++; + } + + ctx->nextToUpdate = target; +} + + + +FORCE_INLINE int Lizard_InsertAndFindBestMatch (Lizard_stream_t* ctx, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos) +{ + U32* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + U32 matchIndex, delta; + const BYTE* match; + int nbAttempts=ctx->params.searchNum; + size_t ml=0; + const int hashLog = ctx->params.hashLog; + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + const U32 maxDistance = (1 << ctx->params.windowLog) - 1; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; + + /* HC4 match finder */ + Lizard_Insert(ctx, ip); + matchIndex = HashTable[LIZARD_HC_HASH_FUNCTION(ip, hashLog)]; + + while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { + nbAttempts--; + if (matchIndex >= dictLimit) { + match = base + matchIndex; +#if LIZARD_HC_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_HC_MIN_OFFSET) +#endif + if (*(match+ml) == *(ip+ml) + && (MEM_read32(match) == MEM_read32(ip))) + { + size_t const mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; +#if LIZARD_HC_LONGOFF_MM > 0 + if ((mlt >= LIZARD_HC_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + if (mlt > ml) { ml = mlt; *matchpos = match; } + } + } else { + match = dictBase + matchIndex; +#if LIZARD_HC_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_HC_MIN_OFFSET) +#endif + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + size_t mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; +#if LIZARD_HC_LONGOFF_MM > 0 + if ((mlt >= LIZARD_HC_LONGOFF_MM) || ((U32)(ip - (base + matchIndex)) < LIZARD_MAX_16BIT_OFFSET)) +#endif + if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ + } + } + delta = DELTANEXT(matchIndex); + if (delta > matchIndex) break; + matchIndex -= delta; + } + + return (int)ml; +} + + +FORCE_INLINE int Lizard_InsertAndGetWiderMatch ( + Lizard_stream_t* ctx, + const BYTE* const ip, + const BYTE* const iLowLimit, + const BYTE* const iHighLimit, + int longest, + const BYTE** matchpos, + const BYTE** startpos) +{ + U32* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + U32 matchIndex, delta; + int nbAttempts = ctx->params.searchNum; + int LLdelta = (int)(ip-iLowLimit); + const int hashLog = ctx->params.hashLog; + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + const U32 maxDistance = (1 << ctx->params.windowLog) - 1; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; + + /* First Match */ + Lizard_Insert(ctx, ip); + matchIndex = HashTable[LIZARD_HC_HASH_FUNCTION(ip, hashLog)]; + + while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { + nbAttempts--; + if (matchIndex >= dictLimit) { + const BYTE* match = base + matchIndex; +#if LIZARD_HC_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_HC_MIN_OFFSET) +#endif + if (*(iLowLimit + longest) == *(match - LLdelta + longest)) { + if (MEM_read32(match) == MEM_read32(ip)) { + int mlt = MINMATCH + Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + int back = 0; + while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + +#if LIZARD_HC_LONGOFF_MM > 0 + if ((mlt >= LIZARD_HC_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) +#endif + if (mlt > longest) { + longest = (int)mlt; + *matchpos = match+back; + *startpos = ip+back; + } + } + } + } else { + const BYTE* match = dictBase + matchIndex; +#if LIZARD_HC_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_HC_MIN_OFFSET) +#endif + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + int back=0; + size_t mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; + while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; + mlt -= back; +#if LIZARD_HC_LONGOFF_MM > 0 + if ((mlt >= LIZARD_HC_LONGOFF_MM) || ((U32)(ip - (base + matchIndex)) < LIZARD_MAX_16BIT_OFFSET)) +#endif + if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } + } + } + delta = DELTANEXT(matchIndex); + if (delta > matchIndex) break; + matchIndex -= delta; + } + + return longest; +} + + +FORCE_INLINE int Lizard_compress_hashChain ( + Lizard_stream_t* const ctx, + const BYTE* ip, + const BYTE* const iend) +{ + const BYTE* anchor = ip; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + + int ml, ml2, ml3, ml0; + const BYTE* ref = NULL; + const BYTE* start2 = NULL; + const BYTE* ref2 = NULL; + const BYTE* start3 = NULL; + const BYTE* ref3 = NULL; + const BYTE* start0; + const BYTE* ref0; + + /* init */ + ip++; + + /* Main Loop */ + while (ip < mflimit) { + ml = Lizard_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref)); + if (!ml) { ip++; continue; } + + /* saved, in case we would skip too much */ + start0 = ip; + ref0 = ref; + ml0 = ml; + +_Search2: + if (ip+ml < mflimit) + ml2 = Lizard_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2); + else ml2 = ml; + + if (ml2 == ml) { /* No better match */ + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + continue; + } + + if (start0 < ip) { + if (start2 < ip + ml0) { /* empirical */ + ip = start0; + ref = ref0; + ml = ml0; + } + } + + /* Here, start0==ip */ + if ((start2 - ip) < 3) { /* First Match too small : removed */ + ml = ml2; + ip = start2; + ref =ref2; + goto _Search2; + } + +_Search3: + /* + * Currently we have : + * ml2 > ml1, and + * ip1+3 <= ip2 (usually < ip1+ml1) + */ + if ((start2 - ip) < OPTIMAL_ML) { + int correction; + int new_ml = ml; + if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; + if (ip+new_ml > start2 + ml2 - GET_MINMATCH((U32)(start2 - ref2))) { + new_ml = (int)(start2 - ip) + ml2 - GET_MINMATCH((U32)(start2 - ref2)); + if (new_ml < GET_MINMATCH((U32)(ip - ref))) { // match2 doesn't fit + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + continue; + } + } + correction = new_ml - (int)(start2 - ip); + if (correction > 0) { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } + /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ + + if (start2 + ml2 < mflimit) + ml3 = Lizard_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3); + else ml3 = ml2; + + if (ml3 == ml2) { /* No better match : 2 sequences to encode */ + /* ip & ref are known; Now for ml */ + if (start2 < ip+ml) ml = (int)(start2 - ip); + /* Now, encode 2 sequences */ + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + ip = start2; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml2, ref2)) return 0; + continue; + } + + if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */ + if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ + if (start2 < ip+ml) { + int correction = (int)(ip+ml - start2); + start2 += correction; + ref2 += correction; + ml2 -= correction; + if (ml2 < GET_MINMATCH((U32)(start2 - ref2))) { + start2 = start3; + ref2 = ref3; + ml2 = ml3; + } + } + + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + ip = start3; + ref = ref3; + ml = ml3; + + start0 = start2; + ref0 = ref2; + ml0 = ml2; + goto _Search2; + } + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + goto _Search3; + } + + /* + * OK, now we have 3 ascending matches; let's write at least the first one + * ip & ref are known; Now for ml + */ + if (start2 < ip+ml) { + if ((start2 - ip) < (int)ML_MASK_LZ4) { + int correction; + if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; + if (ip + ml > start2 + ml2 - GET_MINMATCH((U32)(start2 - ref2))) { + ml = (int)(start2 - ip) + ml2 - GET_MINMATCH((U32)(start2 - ref2)); + if (ml < GET_MINMATCH((U32)(ip - ref))) { // match2 doesn't fit, remove it + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + ip = start3; + ref = ref3; + ml = ml3; + + start0 = start2; + ref0 = ref2; + ml0 = ml2; + goto _Search2; + } + } + correction = ml - (int)(start2 - ip); + if (correction > 0) { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } else { + ml = (int)(start2 - ip); + } + } + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + + ip = start2; + ref = ref2; + ml = ml2; + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + + goto _Search3; + } + + /* Encode Last Literals */ + ip = iend; + if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; + + /* End */ + return 1; +_output_error: + return 0; +} diff --git a/contrib/lizard/lib/lizard_parser_lowestprice.h b/contrib/lizard/lib/lizard_parser_lowestprice.h new file mode 100644 index 00000000000..f8b95dad4ea --- /dev/null +++ b/contrib/lizard/lib/lizard_parser_lowestprice.h @@ -0,0 +1,376 @@ +#define LIZARD_LOWESTPRICE_MIN_OFFSET 8 + + +FORCE_INLINE size_t Lizard_more_profitable(Lizard_stream_t* const ctx, const BYTE *best_ip, size_t best_off, size_t best_common, const BYTE *ip, size_t off, size_t common, size_t literals, int last_off) +{ + size_t sum; + + if (literals > 0) + sum = MAX(common + literals, best_common); + else + sum = MAX(common, best_common - literals); + + if ((int)off == last_off) off = 0; // rep code + if ((int)best_off == last_off) best_off = 0; + + return Lizard_get_price_LIZv1(ctx, last_off, ip, ctx->off24pos, sum - common, (U32)off, common) <= Lizard_get_price_LIZv1(ctx, last_off, best_ip, ctx->off24pos, sum - best_common, (U32)best_off, best_common); +} + + +FORCE_INLINE size_t Lizard_better_price(Lizard_stream_t* const ctx, const BYTE *best_ip, size_t best_off, size_t best_common, const BYTE *ip, size_t off, size_t common, int last_off) +{ + if ((int)off == last_off) off = 0; // rep code + if ((int)best_off == last_off) best_off = 0; + + return Lizard_get_price_LIZv1(ctx, last_off, ip, ctx->off24pos, 0, (U32)off, common) < Lizard_get_price_LIZv1(ctx, last_off, best_ip, ctx->off24pos, common - best_common, (U32)best_off, best_common); +} + + +FORCE_INLINE int Lizard_FindMatchLowestPrice (Lizard_stream_t* ctx, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos) +{ + U32* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const intptr_t dictLimit = ctx->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1; + const intptr_t current = (ip - base); + const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance; + const BYTE* const lowPrefixPtr = base + dictLimit; + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + const size_t minMatchLongOff = ctx->params.minMatchLongOff; + intptr_t matchIndex; + const BYTE* match, *matchDict; + int nbAttempts=ctx->params.searchNum; + size_t ml=0, mlt; + + matchIndex = HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + + if (ctx->last_off >= LIZARD_LOWESTPRICE_MIN_OFFSET) { + intptr_t matchIndexLO = (ip - ctx->last_off) - base; + if (matchIndexLO >= lowLimit) { + if (matchIndexLO >= dictLimit) { + match = base + matchIndexLO; + mlt = Lizard_count(ip, match, iLimit);// + MINMATCH; + // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) + if (mlt > REPMINMATCH) { + *matchpos = match; + return (int)mlt; + } + } else { + match = dictBase + matchIndexLO; + if ((U32)((dictLimit-1) - matchIndexLO) >= 3) { /* intentional overflow */ + mlt = Lizard_count_2segments(ip, match, iLimit, dictEnd, lowPrefixPtr); + // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) + if (mlt > REPMINMATCH) { + *matchpos = base + matchIndexLO; /* virtual matchpos */ + return (int)mlt; + } + } + } + } + } + + +#if MINMATCH == 3 + { + U32 matchIndex3 = ctx->hashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; + if (matchIndex3 < current && matchIndex3 >= lowLimit) + { + size_t offset = (size_t)current - matchIndex3; + if (offset < LIZARD_MAX_8BIT_OFFSET) + { + match = ip - offset; + if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) + { + ml = 3;//Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + *matchpos = match; + } + } + } + } +#endif + while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { + nbAttempts--; + match = base + matchIndex; + if ((U32)(ip - match) >= LIZARD_LOWESTPRICE_MIN_OFFSET) { + if (matchIndex >= dictLimit) { + if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) { + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (!ml || (mlt > ml && Lizard_better_price(ctx, ip, (ip - *matchpos), ml, ip, (ip - match), mlt, ctx->last_off))) + { ml = mlt; *matchpos = match; } + } + } else { + matchDict = dictBase + matchIndex; + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(matchDict) == MEM_read32(ip)) { + mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (!ml || (mlt > ml && Lizard_better_price(ctx, ip, (ip - *matchpos), ml, ip, (U32)(ip - match), mlt, ctx->last_off))) + { ml = mlt; *matchpos = match; } /* virtual matchpos */ + } + } + } + matchIndex -= chainTable[matchIndex & contentMask]; + } + + return (int)ml; +} + + +FORCE_INLINE size_t Lizard_GetWiderMatch ( + Lizard_stream_t* ctx, + const BYTE* const ip, + const BYTE* const iLowLimit, + const BYTE* const iHighLimit, + size_t longest, + const BYTE** matchpos, + const BYTE** startpos) +{ + U32* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const intptr_t dictLimit = ctx->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1; + const intptr_t current = (ip - base); + const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance; + const BYTE* const lowPrefixPtr = base + dictLimit; + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + const BYTE* match, *matchDict; + const size_t minMatchLongOff = ctx->params.minMatchLongOff; + intptr_t matchIndex; + int nbAttempts = ctx->params.searchNum; + size_t mlt; + + /* First Match */ + matchIndex = HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + + if (ctx->last_off >= LIZARD_LOWESTPRICE_MIN_OFFSET) { + intptr_t matchIndexLO = (ip - ctx->last_off) - base; + if (matchIndexLO >= lowLimit) { + if (matchIndexLO >= dictLimit) { + match = base + matchIndexLO; + if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) { + int back = 0; + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; + while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + + if (mlt > longest) + if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) { + *matchpos = match+back; + *startpos = ip+back; + longest = mlt; + } + } + } else { + match = dictBase + matchIndexLO; + if ((U32)((dictLimit-1) - matchIndexLO) >= 3) /* intentional overflow */ + if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) { + int back=0; + mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; + while ((ip+back > iLowLimit) && (matchIndexLO+back > lowLimit) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + + if (mlt > longest) + if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) { + *matchpos = base + matchIndexLO + back; /* virtual matchpos */ + *startpos = ip+back; + longest = mlt; + } + } + } + } + } + +#if MINMATCH == 3 + { + U32 matchIndex3 = ctx->hashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; + if (matchIndex3 < current && matchIndex3 >= lowLimit) { + size_t offset = (size_t)current - matchIndex3; + if (offset < LIZARD_MAX_8BIT_OFFSET) { + match = ip - offset; + if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) { + mlt = Lizard_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; + + int back = 0; + while ((ip + back > iLowLimit) && (match + back > lowPrefixPtr) && (ip[back - 1] == match[back - 1])) back--; + mlt -= back; + + if (!longest || (mlt > longest && Lizard_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (ip - match), mlt, ctx->last_off))) { + *matchpos = match + back; + *startpos = ip + back; + longest = mlt; + } + } + } + } + } +#endif + + while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { + nbAttempts--; + match = base + matchIndex; + if ((U32)(ip - match) >= LIZARD_LOWESTPRICE_MIN_OFFSET) { + if (matchIndex >= dictLimit) { + if (MEM_read32(match) == MEM_read32(ip)) { + int back = 0; + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; + while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (!longest || (mlt > longest && Lizard_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (ip - match), mlt, ctx->last_off))) + { longest = mlt; *startpos = ip+back; *matchpos = match+back; } + } + } else { + matchDict = dictBase + matchIndex; + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(matchDict) == MEM_read32(ip)) { + int back=0; + mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; + while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchDict[back-1])) back--; + mlt -= back; + + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (!longest || (mlt > longest && Lizard_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (U32)(ip - match), mlt, ctx->last_off))) + { longest = mlt; *startpos = ip+back; *matchpos = match+back; } /* virtual matchpos */ + } + } + } + matchIndex -= chainTable[matchIndex & contentMask]; + } + + return longest; +} + + + + +FORCE_INLINE int Lizard_compress_lowestPrice( + Lizard_stream_t* const ctx, + const BYTE* ip, + const BYTE* const iend) +{ + const BYTE* anchor = ip; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + + size_t ml, ml2, ml0; + const BYTE* ref=NULL; + const BYTE* start2=NULL; + const BYTE* ref2=NULL; + const BYTE* start0; + const BYTE* ref0; + const BYTE* lowPrefixPtr = ctx->base + ctx->dictLimit; + const size_t minMatchLongOff = ctx->params.minMatchLongOff; + const size_t sufficient_len = ctx->params.sufficientLength; + + /* Main Loop */ + while (ip < mflimit) + { + Lizard_Insert(ctx, ip); + ml = Lizard_FindMatchLowestPrice (ctx, ip, matchlimit, (&ref)); + if (!ml) { ip++; continue; } + + { + int back = 0; + while ((ip + back > anchor) && (ref + back > lowPrefixPtr) && (ip[back - 1] == ref[back - 1])) back--; + ml -= back; + ip += back; + ref += back; + } + + /* saved, in case we would skip too much */ + start0 = ip; + ref0 = ref; + ml0 = ml; + // goto _Encode; + +_Search: + if (ip+ml >= mflimit) { goto _Encode; } + if (ml >= sufficient_len) { goto _Encode; } + + Lizard_Insert(ctx, ip); + ml2 = (int)Lizard_GetWiderMatch(ctx, ip + ml - 2, anchor, matchlimit, 0, &ref2, &start2); + if (!ml2) goto _Encode; + + { + U64 price, best_price; + int off0=0, off1=0; + const BYTE *pos, *best_pos; + + // find the lowest price for encoding ml bytes + best_pos = ip; + best_price = LIZARD_MAX_PRICE; + off0 = (int)(ip - ref); + off1 = (int)(start2 - ref2); + + for (pos = ip + ml; pos >= start2; pos--) + { + int common0 = (int)(pos - ip); + if (common0 >= MINMATCH) { + price = (int)Lizard_get_price_LIZv1(ctx, ctx->last_off, ip, ctx->off24pos, ip - anchor, (off0 == ctx->last_off) ? 0 : off0, common0); + + { + int common1 = (int)(start2 + ml2 - pos); + if (common1 >= MINMATCH) + price += Lizard_get_price_LIZv1(ctx, ctx->last_off, pos, ctx->off24pos, 0, (off1 == off0) ? 0 : (off1), common1); + else + price += Lizard_get_price_LIZv1(ctx, ctx->last_off, pos, ctx->off24pos, common1, 0, 0); + } + + if (price < best_price) { + best_price = price; + best_pos = pos; + } + } else { + price = Lizard_get_price_LIZv1(ctx, ctx->last_off, ip, ctx->off24pos, start2 - anchor, (off1 == ctx->last_off) ? 0 : off1, ml2); + + if (price < best_price) + best_pos = pos; + break; + } + } + ml = (int)(best_pos - ip); + } + + + if ((ml < MINMATCH) || ((ml < minMatchLongOff) && ((U32)(ip-ref) >= LIZARD_MAX_16BIT_OFFSET))) + { + ip = start2; + ref = ref2; + ml = ml2; + goto _Search; + } + +_Encode: + if (start0 < ip) + { + if (Lizard_more_profitable(ctx, ip, (ip - ref), ml, start0, (start0 - ref0), ml0, (ref0 - ref), ctx->last_off)) + { + ip = start0; + ref = ref0; + ml = ml0; + } + } + + if (Lizard_encodeSequence_LIZv1(ctx, &ip, &anchor, ml, ((ip - ref == ctx->last_off) ? ip : ref))) return 0; + } + + /* Encode Last Literals */ + ip = iend; + if (Lizard_encodeLastLiterals_LIZv1(ctx, &ip, &anchor)) goto _output_error; + + /* End */ + return 1; +_output_error: + return 0; +} + diff --git a/contrib/lizard/lib/lizard_parser_nochain.h b/contrib/lizard/lib/lizard_parser_nochain.h new file mode 100644 index 00000000000..e8abb586ff4 --- /dev/null +++ b/contrib/lizard/lib/lizard_parser_nochain.h @@ -0,0 +1,318 @@ +#define OPTIMAL_ML (int)((ML_MASK_LZ4-1)+MINMATCH) + +//#define LIZARD_NOCHAIN_HASH_FUNCTION(ip, hashLog) Lizard_hashPtr(ip, hashLog, ctx->params.searchLength) +#define LIZARD_NOCHAIN_HASH_FUNCTION(ip, hashLog) Lizard_hash5Ptr(ip, hashLog) +#define LIZARD_NOCHAIN_MIN_OFFSET 8 + +/* Update chains up to ip (excluded) */ +FORCE_INLINE void Lizard_InsertNoChain (Lizard_stream_t* ctx, const BYTE* ip) +{ + U32* const hashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + U32 const target = (U32)(ip - base); + U32 idx = ctx->nextToUpdate; + const int hashLog = ctx->params.hashLog; + + while (idx < target) { + size_t const h = LIZARD_NOCHAIN_HASH_FUNCTION(base+idx, hashLog); + if ((hashTable[h] >= idx) || (idx >= hashTable[h] + LIZARD_NOCHAIN_MIN_OFFSET)) + hashTable[h] = idx; + idx++; + } + + ctx->nextToUpdate = target; +} + + +FORCE_INLINE int Lizard_InsertAndFindBestMatchNoChain (Lizard_stream_t* ctx, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos) +{ + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + U32 matchIndex; + const BYTE* match; + size_t ml=0; + const int hashLog = ctx->params.hashLog; + const U32 maxDistance = (1 << ctx->params.windowLog) - 1; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; + + /* HC4 match finder */ + Lizard_InsertNoChain(ctx, ip); + matchIndex = HashTable[LIZARD_NOCHAIN_HASH_FUNCTION(ip, hashLog)]; + + if ((matchIndex < current) && (matchIndex >= lowLimit)) { + if (matchIndex >= dictLimit) { + match = base + matchIndex; +#if LIZARD_NOCHAIN_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_NOCHAIN_MIN_OFFSET) +#endif + if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) + { + size_t const mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if (mlt > ml) { ml = mlt; *matchpos = match; } + } + } else { + match = dictBase + matchIndex; +#if LIZARD_NOCHAIN_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_NOCHAIN_MIN_OFFSET) +#endif + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + size_t mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; + if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ + } + } + } + + return (int)ml; +} + + +FORCE_INLINE int Lizard_InsertAndGetWiderMatchNoChain ( + Lizard_stream_t* ctx, + const BYTE* const ip, + const BYTE* const iLowLimit, + const BYTE* const iHighLimit, + int longest, + const BYTE** matchpos, + const BYTE** startpos) +{ + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + U32 matchIndex; + int LLdelta = (int)(ip-iLowLimit); + const int hashLog = ctx->params.hashLog; + const U32 maxDistance = (1 << ctx->params.windowLog) - 1; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; + + /* First Match */ + Lizard_InsertNoChain(ctx, ip); + matchIndex = HashTable[LIZARD_NOCHAIN_HASH_FUNCTION(ip, hashLog)]; + + if ((matchIndex < current) && (matchIndex >= lowLimit)) { + if (matchIndex >= dictLimit) { + const BYTE* match = base + matchIndex; +#if LIZARD_NOCHAIN_MIN_OFFSET > 0 + if ((U32)(ip - match) >= LIZARD_NOCHAIN_MIN_OFFSET) +#endif + if (*(iLowLimit + longest) == *(match - LLdelta + longest)) { + if (MEM_read32(match) == MEM_read32(ip)) { + int mlt = MINMATCH + Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + int back = 0; + while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + + if (mlt > longest) { + longest = (int)mlt; + *matchpos = match+back; + *startpos = ip+back; + } + } + } + } else { + const BYTE* match = dictBase + matchIndex; +#if LIZARD_NOCHAIN_MIN_OFFSET > 0 + if ((U32)(ip - (base + matchIndex)) >= LIZARD_NOCHAIN_MIN_OFFSET) +#endif + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(match) == MEM_read32(ip)) { + int back=0; + size_t mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; + while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } + } + } + } + + return longest; +} + + +FORCE_INLINE int Lizard_compress_noChain ( + Lizard_stream_t* const ctx, + const BYTE* ip, + const BYTE* const iend) +{ + const BYTE* anchor = ip; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + + int ml, ml2, ml3, ml0; + const BYTE* ref = NULL; + const BYTE* start2 = NULL; + const BYTE* ref2 = NULL; + const BYTE* start3 = NULL; + const BYTE* ref3 = NULL; + const BYTE* start0; + const BYTE* ref0; + + /* init */ + ip++; + + /* Main Loop */ + while (ip < mflimit) { + ml = Lizard_InsertAndFindBestMatchNoChain (ctx, ip, matchlimit, (&ref)); + if (!ml) { ip++; continue; } + + /* saved, in case we would skip too much */ + start0 = ip; + ref0 = ref; + ml0 = ml; + +_Search2: + if (ip+ml < mflimit) + ml2 = Lizard_InsertAndGetWiderMatchNoChain(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2); + else ml2 = ml; + + if (ml2 == ml) { /* No better match */ + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + continue; + } + + if (start0 < ip) { + if (start2 < ip + ml0) { /* empirical */ + ip = start0; + ref = ref0; + ml = ml0; + } + } + + /* Here, start0==ip */ + if ((start2 - ip) < 3) { /* First Match too small : removed */ + ml = ml2; + ip = start2; + ref =ref2; + goto _Search2; + } + +_Search3: + /* + * Currently we have : + * ml2 > ml1, and + * ip1+3 <= ip2 (usually < ip1+ml1) + */ + if ((start2 - ip) < OPTIMAL_ML) { + int correction; + int new_ml = ml; + if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; + if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; + correction = new_ml - (int)(start2 - ip); + if (correction > 0) { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } + /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ + + if (start2 + ml2 < mflimit) + ml3 = Lizard_InsertAndGetWiderMatchNoChain(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3); + else ml3 = ml2; + + if (ml3 == ml2) { /* No better match : 2 sequences to encode */ + /* ip & ref are known; Now for ml */ + if (start2 < ip+ml) ml = (int)(start2 - ip); + /* Now, encode 2 sequences */ + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + ip = start2; + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml2, ref2)) return 0; + continue; + } + + if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */ + if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ + if (start2 < ip+ml) { + int correction = (int)(ip+ml - start2); + start2 += correction; + ref2 += correction; + ml2 -= correction; + if (ml2 < MINMATCH) { + start2 = start3; + ref2 = ref3; + ml2 = ml3; + } + } + + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + ip = start3; + ref = ref3; + ml = ml3; + + start0 = start2; + ref0 = ref2; + ml0 = ml2; + goto _Search2; + } + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + goto _Search3; + } + + /* + * OK, now we have 3 ascending matches; let's write at least the first one + * ip & ref are known; Now for ml + */ + if (start2 < ip+ml) { + if ((start2 - ip) < (int)ML_MASK_LZ4) { + int correction; + if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; + if (ip + ml > start2 + ml2 - MINMATCH) { + ml = (int)(start2 - ip) + ml2 - MINMATCH; + if (ml < MINMATCH) { // match2 doesn't fit, remove it + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + ip = start3; + ref = ref3; + ml = ml3; + + start0 = start2; + ref0 = ref2; + ml0 = ml2; + goto _Search2; + } + } + correction = ml - (int)(start2 - ip); + if (correction > 0) { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } else { + ml = (int)(start2 - ip); + } + } + if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; + + ip = start2; + ref = ref2; + ml = ml2; + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + + goto _Search3; + } + + /* Encode Last Literals */ + ip = iend; + if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; + + /* End */ + return 1; +_output_error: + return 0; +} diff --git a/contrib/lizard/lib/lizard_parser_optimal.h b/contrib/lizard/lib/lizard_parser_optimal.h new file mode 100644 index 00000000000..8c9b83e9590 --- /dev/null +++ b/contrib/lizard/lib/lizard_parser_optimal.h @@ -0,0 +1,679 @@ +#define LIZARD_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LIZARD_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LIZARD_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) + +#define LIZARD_OPTIMAL_MIN_OFFSET 8 +#define LIZARD_OPT_NUM (1<<12) +#define REPMINMATCH 1 + + +FORCE_INLINE size_t Lizard_get_price(Lizard_stream_t* const ctx, int rep, const BYTE *ip, const BYTE *off24pos, size_t litLength, U32 offset, size_t matchLength) +{ + if (ctx->params.decompressType == Lizard_coderwords_LZ4) + return Lizard_get_price_LZ4(ctx, ip, litLength, offset, matchLength); + + return Lizard_get_price_LIZv1(ctx, rep, ip, off24pos, litLength, offset, matchLength); +} + + + +typedef struct +{ + int off; + int len; + int back; +} Lizard_match_t; + +typedef struct +{ + int price; + int off; + int mlen; + int litlen; + int rep; + const BYTE* off24pos; +} Lizard_optimal_t; + + +/* Update chains up to ip (excluded) */ +FORCE_INLINE void Lizard_BinTree_Insert(Lizard_stream_t* ctx, const BYTE* ip) +{ +#if MINMATCH == 3 + U32* HashTable3 = ctx->hashTable3; + const BYTE* const base = ctx->base; + const U32 target = (U32)(ip - base); + U32 idx = ctx->nextToUpdate; + + while(idx < target) { + HashTable3[Lizard_hash3Ptr(base+idx, ctx->params.hashLog3)] = idx; + idx++; + } + + ctx->nextToUpdate = target; +#else + (void)ctx; (void)ip; +#endif +} + + + +FORCE_INLINE int Lizard_GetAllMatches ( + Lizard_stream_t* ctx, + const BYTE* const ip, + const BYTE* const iLowLimit, + const BYTE* const iHighLimit, + size_t best_mlen, + Lizard_match_t* matches) +{ + U32* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const intptr_t dictLimit = ctx->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1; + const intptr_t current = (ip - base); + const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance; + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + const BYTE* match, *matchDict; + const size_t minMatchLongOff = ctx->params.minMatchLongOff; + intptr_t matchIndex; + int nbAttempts = ctx->params.searchNum; + // bool fullSearch = (ctx->params.fullSearch >= 2); + int mnum = 0; + U32* HashPos; + size_t mlt; + + if (ip + MINMATCH > iHighLimit) return 0; + + /* First Match */ + HashPos = &HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + matchIndex = *HashPos; +#if MINMATCH == 3 + { + U32* const HashTable3 = ctx->hashTable3; + U32* HashPos3 = &HashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; + + if ((*HashPos3 < current) && (*HashPos3 >= lowLimit)) { + size_t offset = current - *HashPos3; + if (offset < LIZARD_MAX_8BIT_OFFSET) { + match = ip - offset; + if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) { + size_t mlt = Lizard_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; + + int back = 0; + while ((ip + back > iLowLimit) && (match + back > lowPrefixPtr) && (ip[back - 1] == match[back - 1])) back--; + mlt -= back; + + matches[mnum].off = (int)offset; + matches[mnum].len = (int)mlt; + matches[mnum].back = -back; + mnum++; + } + } + } + + *HashPos3 = current; + } +#endif + + chainTable[current & contentMask] = (U32)(current - matchIndex); + *HashPos = (U32)current; + ctx->nextToUpdate++; + + if (best_mlen < MINMATCH-1) best_mlen = MINMATCH-1; + + while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { + nbAttempts--; + match = base + matchIndex; + if ((U32)(ip - match) >= LIZARD_OPTIMAL_MIN_OFFSET) { + if (matchIndex >= dictLimit) { + if ((/*fullSearch ||*/ ip[best_mlen] == match[best_mlen]) && (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip))) { + int back = 0; + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; + while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (mlt > best_mlen) { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = -back; + mnum++; + + if (best_mlen > LIZARD_OPT_NUM) break; + } + } + } else { + matchDict = dictBase + matchIndex; + // fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match)); + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_readMINMATCH(matchDict) == MEM_readMINMATCH(ip)) { + int back=0; + mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; + while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchDict[back-1])) back--; + mlt -= back; + + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (mlt > best_mlen) { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = -back; + mnum++; + + if (best_mlen > LIZARD_OPT_NUM) break; + } + } + } + } + matchIndex -= chainTable[matchIndex & contentMask]; + } + + return mnum; +} + + + + +FORCE_INLINE int Lizard_BinTree_GetAllMatches ( + Lizard_stream_t* ctx, + const BYTE* const ip, + const BYTE* const iHighLimit, + size_t best_mlen, + Lizard_match_t* matches) +{ + U32* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const intptr_t dictLimit = ctx->dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + const BYTE* const lowPrefixPtr = base + dictLimit; + const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1; + const intptr_t current = (ip - base); + const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance; + const BYTE* match; + const size_t minMatchLongOff = ctx->params.minMatchLongOff; + int nbAttempts = ctx->params.searchNum; + int mnum = 0; + U32 *ptr0, *ptr1, delta0, delta1; + intptr_t matchIndex; + size_t mlt = 0; + U32* HashPos; + + if (ip + MINMATCH > iHighLimit) return 0; + + /* First Match */ + HashPos = &HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + matchIndex = *HashPos; + + +#if MINMATCH == 3 + { + U32* HashPos3 = &ctx->hashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; + + if ((*HashPos3 < current) && (*HashPos3 >= lowLimit)) { + size_t offset = current - *HashPos3; + if (offset < LIZARD_MAX_8BIT_OFFSET) { + match = ip - offset; + if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) + { + mlt = Lizard_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; + + matches[mnum].off = (int)offset; + matches[mnum].len = (int)mlt; + matches[mnum].back = 0; + mnum++; + } + } + *HashPos3 = current; + } + } +#endif + + *HashPos = (U32)current; + ctx->nextToUpdate++; + + // check rest of matches + ptr0 = &chainTable[(current*2+1) & contentMask]; + ptr1 = &chainTable[(current*2) & contentMask]; + delta0 = delta1 = (U32)(current - matchIndex); + + if (best_mlen < MINMATCH-1) best_mlen = MINMATCH-1; + + while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { + nbAttempts--; + if (matchIndex >= dictLimit) { + match = base + matchIndex; + // if (ip[mlt] == match[mlt]) + mlt = Lizard_count(ip, match, iHighLimit); + } else { + match = dictBase + matchIndex; + mlt = Lizard_count_2segments(ip, match, iHighLimit, dictEnd, lowPrefixPtr); + if (matchIndex + (int)mlt >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[mlt] */ + } + + if ((U32)(current - matchIndex) >= LIZARD_OPTIMAL_MIN_OFFSET) { + if ((mlt >= minMatchLongOff) || ((U32)(current - matchIndex) < LIZARD_MAX_16BIT_OFFSET)) + if (mlt > best_mlen) { + best_mlen = mlt; + matches[mnum].off = (int)(current - matchIndex); + matches[mnum].len = (int)mlt; + matches[mnum].back = 0; + mnum++; + + if (mlt > LIZARD_OPT_NUM) break; + if (ip + mlt >= iHighLimit) break; + } + } else { +#if 1 + intptr_t newMatchIndex; + size_t newml = 0, newoff = 0; + do { + newoff += (int)(current - matchIndex); + } while (newoff < LIZARD_OPTIMAL_MIN_OFFSET); + newMatchIndex = current - newoff; + if (newMatchIndex >= dictLimit) newml = Lizard_count(ip, base + newMatchIndex, iHighLimit); + + // printf("%d: off=%d mlt=%d\n", (U32)current, (U32)(current - matchIndex), (int)mlt); + // printf("%d: newoff=%d newml=%d\n", (U32)current, (int)newoff, (int)newml); + + if ((newml >= minMatchLongOff) && (newml > best_mlen)) { + best_mlen = newml; + matches[mnum].off = (int)newoff; + matches[mnum].len = (int)newml; + matches[mnum].back = 0; + mnum++; + + if (newml > LIZARD_OPT_NUM) break; + if (ip + newml >= iHighLimit) break; + } +#endif + } + + if (ip[mlt] < match[mlt]) { + *ptr0 = delta0; + ptr0 = &chainTable[(matchIndex*2) & contentMask]; + if (*ptr0 == (U32)-1) break; + delta0 = *ptr0; + delta1 += delta0; + matchIndex -= delta0; + } else { + *ptr1 = delta1; + ptr1 = &chainTable[(matchIndex*2+1) & contentMask]; + if (*ptr1 == (U32)-1) break; + delta1 = *ptr1; + delta0 += delta1; + matchIndex -= delta1; + } + } + + *ptr0 = (U32)-1; + *ptr1 = (U32)-1; + + return mnum; +} + + +#define SET_PRICE(pos, mlen, offset, litlen, price) \ + { \ + while (last_pos < pos) { opt[last_pos+1].price = LIZARD_MAX_PRICE; last_pos++; } \ + opt[pos].mlen = (int)mlen; \ + opt[pos].off = (int)offset; \ + opt[pos].litlen = (int)litlen; \ + opt[pos].price = (int)price; \ + LIZARD_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ + } + + +FORCE_INLINE int Lizard_compress_optimalPrice( + Lizard_stream_t* const ctx, + const BYTE* ip, + const BYTE* const iend) +{ + Lizard_optimal_t opt[LIZARD_OPT_NUM + 4]; + Lizard_match_t matches[LIZARD_OPT_NUM + 1]; + const BYTE *inr; + size_t res, cur, cur2, skip_num = 0; + size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; + + const BYTE* anchor = ip; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const intptr_t dictLimit = ctx->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const intptr_t lowLimit = ctx->lowLimit; + const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1; + + const size_t sufficient_len = ctx->params.sufficientLength; + const int faster_get_matches = (ctx->params.fullSearch == 0); + const size_t minMatchLongOff = ctx->params.minMatchLongOff; + const int lizardOptimalMinOffset = (ctx->params.decompressType == Lizard_coderwords_LZ4) ? (1<<30) : LIZARD_OPTIMAL_MIN_OFFSET; + const size_t repMinMatch = (ctx->params.decompressType == Lizard_coderwords_LZ4) ? MINMATCH : REPMINMATCH; + + /* Main Loop */ + while (ip < mflimit) { + memset(opt, 0, sizeof(Lizard_optimal_t)); + last_pos = 0; + llen = ip - anchor; + + /* check rep code */ + + if (ctx->last_off >= lizardOptimalMinOffset) { + intptr_t matchIndexLO = (ip - ctx->last_off) - base; + mlen = 0; + if ((matchIndexLO >= lowLimit) && (base + matchIndexLO + maxDistance >= ip)) { + if (matchIndexLO >= dictLimit) { + mlen = Lizard_count(ip, base + matchIndexLO, matchlimit); + } else { + mlen = Lizard_count_2segments(ip, dictBase + matchIndexLO, matchlimit, dictEnd, lowPrefixPtr); + } + } + if (mlen >= REPMINMATCH) { + if (mlen > sufficient_len || mlen >= LIZARD_OPT_NUM) { + best_mlen = mlen; best_off = 0; cur = 0; last_pos = 1; + goto encode; + } + + do + { + litlen = 0; + price = Lizard_get_price(ctx, ctx->last_off, ip, ctx->off24pos, llen, 0, mlen); + if (mlen > last_pos || price < (size_t)opt[mlen].price) + SET_PRICE(mlen, mlen, 0, litlen, price); + mlen--; + } + while (mlen >= REPMINMATCH); + } + } + + if (faster_get_matches && last_pos) + match_num = 0; + else + { + if (ctx->params.parserType == Lizard_parser_optimalPrice) { + Lizard_Insert(ctx, ip); + match_num = Lizard_GetAllMatches(ctx, ip, ip, matchlimit, last_pos, matches); + } else { + Lizard_BinTree_Insert(ctx, ip); + match_num = Lizard_BinTree_GetAllMatches(ctx, ip, matchlimit, last_pos, matches); + } + } + + LIZARD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); + if (!last_pos && !match_num) { ip++; continue; } + + if (match_num && (size_t)matches[match_num-1].len > sufficient_len) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto encode; + } + + // set prices using matches at position = 0 + best_mlen = (last_pos > MINMATCH) ? last_pos : MINMATCH; + + for (i = 0; i < match_num; i++) { + mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; + best_mlen = (matches[i].len < LIZARD_OPT_NUM) ? matches[i].len : LIZARD_OPT_NUM; + LIZARD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); + while (mlen <= best_mlen){ + litlen = 0; + price = Lizard_get_price(ctx, ctx->last_off, ip, ctx->off24pos, llen + litlen, matches[i].off, mlen); + + if ((mlen >= minMatchLongOff) || (matches[i].off < LIZARD_MAX_16BIT_OFFSET)) + if (mlen > last_pos || price < (size_t)opt[mlen].price) + SET_PRICE(mlen, mlen, matches[i].off, litlen, price); + mlen++; + } + } + + if (last_pos < repMinMatch) { ip++; continue; } + + opt[0].off24pos = ctx->off24pos; + opt[0].rep = ctx->last_off; + opt[0].mlen = 1; + opt[0].off = -1; + + // check further positions + for (skip_num = 0, cur = 1; cur <= last_pos; cur++) { + int rep; + inr = ip + cur; + + if (opt[cur-1].off == -1) { // -1 = literals, 0 = rep + litlen = opt[cur-1].litlen + 1; + + if (cur != litlen) { + price = opt[cur - litlen].price + Lizard_get_price(ctx, opt[cur-litlen].rep, inr, ctx->off24pos, litlen, 0, 0); + LIZARD_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); + } else { + price = Lizard_get_price(ctx, ctx->last_off, inr, ctx->off24pos, llen + litlen, 0, 0); + LIZARD_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); + } + } else { + litlen = 1; + price = opt[cur - 1].price + Lizard_get_price(ctx, opt[cur-1].rep, inr, ctx->off24pos, litlen, 0, 0); + LIZARD_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, Lizard_get_price(ctx, rep, inr, ctx->off24pos, litlen, 0, 0)); + } + + mlen = 1; + best_mlen = 0; + LIZARD_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); + + if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + SET_PRICE(cur, mlen, -1, litlen, price); + + if (cur == last_pos) break; + + + + /* set rep code */ + if (opt[cur].off != -1) { + mlen = opt[cur].mlen; + offset = opt[cur].off; + if (offset < 1) { + opt[cur].rep = opt[cur-mlen].rep; + opt[cur].off24pos = opt[cur-mlen].off24pos; + LIZARD_LOG_PARSER("%d: COPYREP1 cur=%d mlen=%d rep=%d\n", (int)(inr-source), cur, mlen, opt[cur-mlen].rep); + } else { + opt[cur].rep = (int)offset; + opt[cur].off24pos = (offset >= LIZARD_MAX_16BIT_OFFSET) ? inr : opt[cur-mlen].off24pos; + LIZARD_LOG_PARSER("%d: COPYREP2 cur=%d offset=%d rep=%d\n", (int)(inr-source), cur, offset, opt[cur].rep); + } + } else { + opt[cur].rep = opt[cur-1].rep; // copy rep + opt[cur].off24pos = opt[cur-1].off24pos; + } + + rep = opt[cur].rep; + LIZARD_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep); + + + /* check rep code */ + if (opt[cur].rep >= lizardOptimalMinOffset) { + intptr_t matchIndexLO = (inr - opt[cur].rep) - base; + mlen = 0; + if ((matchIndexLO >= lowLimit) && (base + matchIndexLO + maxDistance >= inr)) { + if (matchIndexLO >= dictLimit) { + mlen = Lizard_count(inr, base + matchIndexLO, matchlimit); + } else { + mlen = Lizard_count_2segments(inr, dictBase + matchIndexLO, matchlimit, dictEnd, lowPrefixPtr); + } + } + if (mlen >= REPMINMATCH/* && mlen > best_mlen*/) { + LIZARD_LOG_PARSER("%d: try REP rep=%d mlen=%d\n", (int)(inr-source), opt[cur].rep, mlen); + LIZARD_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-source), mlen, 0, opt[cur].rep, cur, opt[cur].off); + + if (mlen > sufficient_len || cur + mlen >= LIZARD_OPT_NUM) { + best_mlen = mlen; + best_off = 0; + LIZARD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-source), sufficient_len, best_mlen, best_off, last_pos); + last_pos = cur + 1; + goto encode; + } + + best_mlen = mlen; + if (faster_get_matches) + skip_num = best_mlen; + + do + { + //if (opt[cur].mlen == 1) + if (opt[cur].off == -1) { + litlen = opt[cur].litlen; + + if (cur != litlen) { + price = opt[cur - litlen].price + Lizard_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen); + LIZARD_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); + } else { + price = Lizard_get_price(ctx, rep, inr, ctx->off24pos, llen + litlen, 0, mlen); + LIZARD_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); + } + } else { + litlen = 0; + price = opt[cur].price + Lizard_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen); + LIZARD_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d getprice=%d\n", (int)(inr-source), price, cur, litlen, Lizard_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen - MINMATCH)); + } + + LIZARD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, 0, price, litlen, cur - litlen, opt[cur - litlen].price); + + if (cur + mlen > last_pos || price <= (size_t)opt[cur + mlen].price) // || ((price == opt[cur + mlen].price) && (opt[cur].mlen == 1) && (cur != litlen))) // at equal price prefer REP instead of MATCH + SET_PRICE(cur + mlen, mlen, 0, litlen, price); + mlen--; + } + while (mlen >= REPMINMATCH); + } + } + + if (faster_get_matches && skip_num > 0) { + skip_num--; + continue; + } + + if (ctx->params.parserType == Lizard_parser_optimalPrice) { + Lizard_Insert(ctx, inr); + match_num = Lizard_GetAllMatches(ctx, inr, ip, matchlimit, best_mlen, matches); + LIZARD_LOG_PARSER("%d: Lizard_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); + } else { + Lizard_BinTree_Insert(ctx, inr); + match_num = Lizard_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); + LIZARD_LOG_PARSER("%d: Lizard_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); + } + + + if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { + cur -= matches[match_num-1].back; + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto encode; + } + + // set prices using matches at position = cur + best_mlen = (best_mlen > MINMATCH) ? best_mlen : MINMATCH; + + for (i = 0; i < match_num; i++) { + mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; + cur2 = cur - matches[i].back; + best_mlen = (cur2 + matches[i].len < LIZARD_OPT_NUM) ? (size_t)matches[i].len : LIZARD_OPT_NUM - cur2; + LIZARD_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); + + if (mlen < (size_t)matches[i].back + 1) + mlen = matches[i].back + 1; + + while (mlen <= best_mlen) { + // if (opt[cur2].mlen == 1) + if (opt[cur2].off == -1) + { + litlen = opt[cur2].litlen; + + if (cur2 != litlen) + price = opt[cur2 - litlen].price + Lizard_get_price(ctx, rep, inr, opt[cur2].off24pos, litlen, matches[i].off, mlen); + else + price = Lizard_get_price(ctx, rep, inr, ctx->off24pos, llen + litlen, matches[i].off, mlen); + } else { + litlen = 0; + price = opt[cur2].price + Lizard_get_price(ctx, rep, inr, opt[cur2].off24pos, litlen, matches[i].off, mlen); + } + + LIZARD_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); + // if (cur2 + mlen > last_pos || ((matches[i].off != opt[cur2 + mlen].off) && (price < opt[cur2 + mlen].price))) + + if ((mlen >= minMatchLongOff) || (matches[i].off < LIZARD_MAX_16BIT_OFFSET)) + if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) + { + SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); + } + + mlen++; + } + } + } // for (skip_num = 0, cur = 1; cur <= last_pos; cur++) + + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + + encode: // cur, last_pos, best_mlen, best_off have to be set + for (i = 1; i <= last_pos; i++) { + LIZARD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep); + } + + LIZARD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off, opt[cur].rep); + + opt[0].mlen = 1; + + while (1) { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = (int)best_mlen; + opt[cur].off = (int)best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + for (i = 0; i <= last_pos;) { + LIZARD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep); + i += opt[i].mlen; + } + + cur = 0; + + while (cur < last_pos) { + LIZARD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep); + mlen = opt[cur].mlen; + // if (mlen == 1) { ip++; cur++; continue; } + if (opt[cur].off == -1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + + LIZARD_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-source), (int)(ip-anchor), (int)(offset), mlen); + res = Lizard_encodeSequence(ctx, &ip, &anchor, mlen, ip - offset); + if (res) return 0; + + LIZARD_LOG_PARSER("%d: offset=%d rep=%d\n", (int)(ip-source), offset, ctx->last_off); + } + } + + /* Encode Last Literals */ + ip = iend; + if (Lizard_encodeLastLiterals(ctx, &ip, &anchor)) goto _output_error; + + /* End */ + return 1; +_output_error: + return 0; +} + diff --git a/contrib/lizard/lib/lizard_parser_pricefast.h b/contrib/lizard/lib/lizard_parser_pricefast.h new file mode 100644 index 00000000000..9818e2c4212 --- /dev/null +++ b/contrib/lizard/lib/lizard_parser_pricefast.h @@ -0,0 +1,250 @@ +#define LIZARD_PRICEFAST_MIN_OFFSET 8 + +FORCE_INLINE int Lizard_FindMatchFast(Lizard_stream_t* ctx, intptr_t matchIndex, intptr_t matchIndex3, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos) +{ + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const intptr_t dictLimit = ctx->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1; + const intptr_t current = (U32)(ip - base); + const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance; + const BYTE* const lowPrefixPtr = base + dictLimit; + const size_t minMatchLongOff = ctx->params.minMatchLongOff; + const BYTE* match, *matchDict; + size_t ml=0, mlt; + + if (ctx->last_off >= LIZARD_PRICEFAST_MIN_OFFSET) { + intptr_t matchIndexLO = (ip - ctx->last_off) - base; + if (matchIndexLO >= lowLimit) { + if (matchIndexLO >= dictLimit) { + match = base + matchIndexLO; + if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) { + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) + { + *matchpos = match; + return (int)mlt; + } + } + } else { + match = dictBase + matchIndexLO; + if ((U32)((dictLimit-1) - matchIndexLO) >= 3) /* intentional overflow */ + if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) { + mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; + // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) + { + *matchpos = base + matchIndexLO; /* virtual matchpos */ + return (int)mlt; + } + } + } + } + } + + +#if MINMATCH == 3 + if (matchIndex3 < current && matchIndex3 >= lowLimit) { + intptr_t offset = current - matchIndex3; + if (offset < LIZARD_MAX_8BIT_OFFSET) { + match = ip - offset; + if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) { + ml = 3;//Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + *matchpos = match; + } + } + } +#else + (void)matchIndex3; +#endif + + if ((matchIndex < current) && (matchIndex >= lowLimit)) { + match = base + matchIndex; + if ((U32)(ip - match) >= LIZARD_PRICEFAST_MIN_OFFSET) { + if (matchIndex >= dictLimit) { + if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) { + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (!ml || (mlt > ml)) // && Lizard_better_price((ip - *matchpos), ml, (ip - match), mlt, ctx->last_off))) + { ml = mlt; *matchpos = match; } + } + } else { + matchDict = dictBase + matchIndex; + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(matchDict) == MEM_read32(ip)) { + mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + if (!ml || (mlt > ml)) // && Lizard_better_price((ip - *matchpos), ml, (U32)(ip - match), mlt, ctx->last_off))) + { ml = mlt; *matchpos = match; } /* virtual matchpos */ + } + } + } + } + + return (int)ml; +} + + +FORCE_INLINE int Lizard_FindMatchFaster (Lizard_stream_t* ctx, U32 matchIndex, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos) +{ + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 maxDistance = (1 << ctx->params.windowLog) - 1; + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; + const BYTE* const lowPrefixPtr = base + dictLimit; + const size_t minMatchLongOff = ctx->params.minMatchLongOff; + const BYTE* match, *matchDict; + size_t ml=0, mlt; + + if (matchIndex < current && matchIndex >= lowLimit) { + match = base + matchIndex; + if ((U32)(ip - match) >= LIZARD_PRICEFAST_MIN_OFFSET) { + if (matchIndex >= dictLimit) { + if (MEM_read32(match) == MEM_read32(ip)) { + mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + { ml = mlt; *matchpos = match; } + } + } else { + matchDict = dictBase + matchIndex; + if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ + if (MEM_read32(matchDict) == MEM_read32(ip)) { + mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; + if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) + { ml = mlt; *matchpos = match; } /* virtual matchpos */ + } + } + } + } + + return (int)ml; +} + + + +FORCE_INLINE int Lizard_compress_priceFast( + Lizard_stream_t* const ctx, + const BYTE* ip, + const BYTE* const iend) +{ + const BYTE* anchor = ip; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + + size_t ml, ml2=0; + const BYTE* ref=NULL; + const BYTE* start2=NULL; + const BYTE* ref2=NULL; + const BYTE* lowPrefixPtr = ctx->base + ctx->dictLimit; + U32* HashTable = ctx->hashTable; +#if MINMATCH == 3 + U32* HashTable3 = ctx->hashTable3; +#endif + const BYTE* const base = ctx->base; + const size_t minMatchLongOff = ctx->params.minMatchLongOff; + U32* HashPos; + + /* init */ + ip++; + + /* Main Loop */ + while (ip < mflimit) + { + HashPos = &HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; +#if MINMATCH == 3 + { + U32* HashPos3 = &HashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; + ml = Lizard_FindMatchFast (ctx, *HashPos, *HashPos3, ip, matchlimit, (&ref)); + *HashPos3 = (U32)(ip - base); + } +#else + ml = Lizard_FindMatchFast (ctx, *HashPos, 0, ip, matchlimit, (&ref)); +#endif + if ((*HashPos >= (U32)(ip - base)) || ((U32)(ip - base) >= *HashPos + LIZARD_PRICEFAST_MIN_OFFSET)) + *HashPos = (U32)(ip - base); + + if (!ml) { ip++; continue; } + if ((int)(ip - ref) == ctx->last_off) { ml2=0; ref=ip; goto _Encode; } + + { + int back = 0; + while ((ip+back>anchor) && (ref+back > lowPrefixPtr) && (ip[back-1] == ref[back-1])) back--; + ml -= back; + ip += back; + ref += back; + } + +_Search: + if (ip+ml >= mflimit) goto _Encode; + + start2 = ip + ml - 2; + HashPos = &HashTable[Lizard_hashPtr(start2, ctx->params.hashLog, ctx->params.searchLength)]; + ml2 = Lizard_FindMatchFaster(ctx, *HashPos, start2, matchlimit, (&ref2)); + if ((*HashPos >= (U32)(start2 - base)) || ((U32)(start2 - base) >= *HashPos + LIZARD_PRICEFAST_MIN_OFFSET)) + *HashPos = (U32)(start2 - base); + + if (!ml2) goto _Encode; + + { + int back = 0; + while ((start2+back>ip) && (ref2+back > lowPrefixPtr) && (start2[back-1] == ref2[back-1])) back--; + ml2 -= back; + start2 += back; + ref2 += back; + } + + if (ml2 <= ml) { ml2 = 0; goto _Encode; } + + if (start2 <= ip) + { + + ip = start2; ref = ref2; ml = ml2; + ml2 = 0; + goto _Encode; + } + + if (start2 - ip < 3) + { + ip = start2; ref = ref2; ml = ml2; + ml2 = 0; + goto _Search; + } + + if (start2 < ip + ml) + { + size_t correction = ml - (int)(start2 - ip); + start2 += correction; + ref2 += correction; + ml2 -= correction; + if (ml2 < 3) { ml2 = 0; } + if ((ml2 < minMatchLongOff) && ((U32)(start2 - ref2) >= LIZARD_MAX_16BIT_OFFSET)) { ml2 = 0; } + } + +_Encode: + if (Lizard_encodeSequence_LIZv1(ctx, &ip, &anchor, ml, ref)) goto _output_error; + + if (ml2) + { + ip = start2; ref = ref2; ml = ml2; + ml2 = 0; + goto _Search; + } + } + + /* Encode Last Literals */ + ip = iend; + if (Lizard_encodeLastLiterals_LIZv1(ctx, &ip, &anchor)) goto _output_error; + + /* End */ + return 1; +_output_error: + return 0; +} + diff --git a/contrib/lizard/lib/xxhash/xxhash.c b/contrib/lizard/lib/xxhash/xxhash.c new file mode 100644 index 00000000000..833b99f3b21 --- /dev/null +++ b/contrib/lizard/lib/xxhash/xxhash.c @@ -0,0 +1,888 @@ +/* +* xxHash - Fast Hash algorithm +* Copyright (C) 2012-2016, Yann Collet +* +* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following disclaimer +* in the documentation and/or other materials provided with the +* distribution. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +* You can contact the author at : +* - xxHash homepage: http://www.xxhash.com +* - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + + +/* ************************************* +* Tuning parameters +***************************************/ +/*!XXH_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. + * It can generate buggy code on targets which do not support unaligned memory accesses. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://stackoverflow.com/a/32095106/646947 for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define XXH_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/*!XXH_ACCEPT_NULL_INPUT_POINTER : + * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. + * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. + * By default, this option is disabled. To enable it, uncomment below define : + */ +/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ + +/*!XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independence be of no importance for your application, you may set the #define below to 1, + * to improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ +#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ +# define XXH_FORCE_NATIVE_FORMAT 0 +#endif + +/*!XXH_FORCE_ALIGN_CHECK : + * This is a minor performance trick, only useful with lots of very small keys. + * It means : check for aligned/unaligned input. + * The check costs one initial branch per hash; + * set it to 0 when the input is guaranteed to be aligned, + * or when alignment doesn't matter for performance. + */ +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/*! Modify the local functions below should you wish to use some other memory routines +* for malloc(), free() */ +#include +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } +/*! and for memcpy() */ +#include +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# define FORCE_INLINE static __forceinline +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/* ************************************* +* Basic Types +***************************************/ +#ifndef MEM_MODULE +# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; +# else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; +# endif +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U32 u32; } __attribute__((packed)) unalign; +static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } + +#else + +/* portable and safe solution. Generally efficient. + * see : http://stackoverflow.com/a/32095106/646947 + */ +static U32 XXH_read32(const void* memPtr) +{ + U32 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* ************************************* +* Architecture Macros +***************************************/ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; + +/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ +#ifndef XXH_CPU_LITTLE_ENDIAN + static const int g_one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) +#endif + + +/* *************************** +* Memory reads +*****************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + else + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +static U32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} + + +/* ************************************* +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bits hash functions +*********************************************************************/ +static const U32 PRIME32_1 = 2654435761U; +static const U32 PRIME32_2 = 2246822519U; +static const U32 PRIME32_3 = 3266489917U; +static const U32 PRIME32_4 = 668265263U; +static const U32 PRIME32_5 = 374761393U; + +static U32 XXH32_round(U32 seed, U32 input) +{ + seed += input * PRIME32_2; + seed = XXH_rotl32(seed, 13); + seed *= PRIME32_1; + return seed; +} + +FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; + v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; + v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; + v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; + } while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p+4<=bEnd) { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, input, len); + return XXH32_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + + +/*====== Hash streaming ======*/ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ + state.v1 = seed + PRIME32_1 + PRIME32_2; + state.v2 = seed + PRIME32_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME32_1; + memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + + +FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len_32 += (unsigned)len; + state->large_len |= (len>=16) | (state->total_len_32>=16); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (unsigned)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const U32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) +{ + const BYTE * p = (const BYTE*)state->mem32; + const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; + U32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } else { + h32 = state->v3 /* == seed */ + PRIME32_5; + } + + h32 += state->total_len_32; + + while (p+4<=bEnd) { + h32 += XXH_readLE32(p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + +/*====== Canonical representation ======*/ + +/*! Default XXH result types are basic unsigned 32 and 64 bits. +* The canonical representation follows human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. +*/ + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bits hash functions +*********************************************************************/ + +/*====== Memory access ======*/ + +#ifndef MEM_MODULE +# define MEM_MODULE +# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t U64; +# else + typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */ +# endif +#endif + + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64; +static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; } + +#else + +/* portable and safe solution. Generally efficient. + * see : http://stackoverflow.com/a/32095106/646947 + */ + +static U64 XXH_read64(const void* memPtr) +{ + U64 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + else + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + +static U64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} + + +/*====== xxh64 ======*/ + +static const U64 PRIME64_1 = 11400714785074694791ULL; +static const U64 PRIME64_2 = 14029467366897019727ULL; +static const U64 PRIME64_3 = 1609587929392839161ULL; +static const U64 PRIME64_4 = 9650029242287828579ULL; +static const U64 PRIME64_5 = 2870177450012600261ULL; + +static U64 XXH64_round(U64 acc, U64 input) +{ + acc += input * PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= PRIME64_1; + return acc; +} + +static U64 XXH64_mergeRound(U64 acc, U64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; +} + +FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; + v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; + v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; + v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; + } while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, input, len); + return XXH64_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +/*====== Hash Streaming ======*/ + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) +{ + XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ + state.v1 = seed + PRIME64_1 + PRIME64_2; + state.v2 = seed + PRIME64_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME64_1; + memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + +FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) +{ + const BYTE * p = (const BYTE*)state->mem64; + const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; + U64 h64; + + if (state->total_len >= 32) { + U64 const v1 = state->v1; + U64 const v2 = state->v2; + U64 const v3 = state->v3; + U64 const v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + } else { + h64 = state->v3 + PRIME64_5; + } + + h64 += (U64) state->total_len; + + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) { + h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + +XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + +/*====== Canonical representation ======*/ + +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#endif /* XXH_NO_LONG_LONG */ diff --git a/contrib/lizard/lib/xxhash/xxhash.h b/contrib/lizard/lib/xxhash/xxhash.h new file mode 100644 index 00000000000..9d831e03b35 --- /dev/null +++ b/contrib/lizard/lib/xxhash/xxhash.h @@ -0,0 +1,293 @@ +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +A 64-bits version, named XXH64, is available since r35. +It offers much better speed, but for 64-bits applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************** +* Definitions +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/* **************************** +* API modifier +******************************/ +/** XXH_PRIVATE_API +* This is useful to include xxhash functions in `static` mode +* in order to inline them, and remove their symbol from the public list. +* Methodology : +* #define XXH_PRIVATE_API +* #include "xxhash.h" +* `xxhash.c` is automatically included. +* It's not useful to compile and link it as a separate module. +*/ +#ifdef XXH_PRIVATE_API +# ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY +# endif +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else +# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ +# endif +#else +# define XXH_PUBLIC_API /* do nothing */ +#endif /* XXH_PRIVATE_API */ + +/*!XXH_NAMESPACE, aka Namespace Emulation : + +If you want to include _and expose_ xxHash functions from within your own library, +but also want to avoid symbol collisions with other libraries which may also include xxHash, + +you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library +with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). + +Note that no change is required within the calling program as long as it includes `xxhash.h` : +regular symbol name will be automatically translated by this header. +*/ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 6 +#define XXH_VERSION_RELEASE 2 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/*-********************************************************************** +* 32-bits hash +************************************************************************/ +typedef unsigned int XXH32_hash_t; + +/*! XXH32() : + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); + +/*====== Streaming ======*/ +typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +/* +These functions generate the xxHash of an input provided in multiple segments. +Note that, for small input, they are slower than single-call functions, due to state management. +For small input, prefer `XXH32()` and `XXH64()` . + +XXH state must first be allocated, using XXH*_createState() . + +Start a new hash by initializing state with a seed, using XXH*_reset(). + +Then, feed the hash state by calling XXH*_update() as many times as necessary. +Obviously, input must be allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + +Finally, a hash value can be produced anytime, by using XXH*_digest(). +This function returns the nn-bits hash as an int or long long. + +It's still possible to continue inserting input into the hash state after a digest, +and generate some new hashes later on, by calling again XXH*_digest(). + +When done, free XXH state space if it was allocated dynamically. +*/ + +/*====== Canonical representation ======*/ + +typedef struct { unsigned char digest[4]; } XXH32_canonical_t; +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + +/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. +* The canonical representation uses human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. +*/ + + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bits hash +************************************************************************/ +typedef unsigned long long XXH64_hash_t; + +/*! XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + "seed" can be used to alter the result predictably. + This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark). +*/ +XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); + +/*====== Streaming ======*/ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/*====== Canonical representation ======*/ +typedef struct { unsigned char digest[8]; } XXH64_canonical_t; +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); +#endif /* XXH_NO_LONG_LONG */ + + +#ifdef XXH_STATIC_LINKING_ONLY + +/* ================================================================================================ + This section contains definitions which are not guaranteed to remain stable. + They may change in future versions, becoming incompatible with a different version of the library. + They shall only be used with static linking. + Never use these definitions in association with dynamic linking ! +=================================================================================================== */ + +/* These definitions are only meant to make possible + static allocation of XXH state, on stack or in a struct for example. + Never use members directly. */ + +struct XXH32_state_s { + unsigned total_len_32; + unsigned large_len; + unsigned v1; + unsigned v2; + unsigned v3; + unsigned v4; + unsigned mem32[4]; /* buffer defined as U32 for alignment */ + unsigned memsize; + unsigned reserved; /* never read nor write, will be removed in a future version */ +}; /* typedef'd to XXH32_state_t */ + +#ifndef XXH_NO_LONG_LONG /* remove 64-bits support */ +struct XXH64_state_s { + unsigned long long total_len; + unsigned long long v1; + unsigned long long v2; + unsigned long long v3; + unsigned long long v4; + unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ + unsigned memsize; + unsigned reserved[2]; /* never read nor write, will be removed in a future version */ +}; /* typedef'd to XXH64_state_t */ +#endif + +#ifdef XXH_PRIVATE_API +# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ +#endif + +#endif /* XXH_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* XXHASH_H_5627135585666179 */ diff --git a/contrib/lizard/programs/.gitignore b/contrib/lizard/programs/.gitignore new file mode 100644 index 00000000000..c2bfcafa9ee --- /dev/null +++ b/contrib/lizard/programs/.gitignore @@ -0,0 +1,15 @@ +# local binary (Makefile) +lizard +lizardc32 +datagen +frametest +frametest32 +fullbench +fullbench32 +fuzzer +fuzzer32 +*.exe + +# tests files +tmp* +_* \ No newline at end of file diff --git a/contrib/lizard/programs/COPYING b/contrib/lizard/programs/COPYING new file mode 100644 index 00000000000..d159169d105 --- /dev/null +++ b/contrib/lizard/programs/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/contrib/lizard/programs/README.md b/contrib/lizard/programs/README.md new file mode 100644 index 00000000000..ee116c116c9 --- /dev/null +++ b/contrib/lizard/programs/README.md @@ -0,0 +1,73 @@ +Command Line Interface for LZ4 library +============================================ + +Command Line Interface (CLI) can be created using the `make` command without any additional parameters. +There are also multiple targets that create different variations of CLI: +- `lizard` : default CLI, with a command line syntax close to gzip +- `lizardc32` : Same as `lizard`, but forced to compile in 32-bits mode + + +#### Aggregation of parameters +CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`. + + + +#### Benchmark in Command Line Interface +CLI includes in-memory compression benchmark module for lizard. +The benchmark is conducted using a given filename. +The file is read into memory. +It makes benchmark more precise as it eliminates I/O overhead. + +The benchmark measures ratio, compressed size, compression and decompression speed. +One can select compression levels starting from `-b` and ending with `-e`. +The `-i` parameter selects a number of seconds used for each of tested levels. + + + +#### Usage of Command Line Interface +The full list of commands can be obtained with `-h` or `-H` parameter: +``` +Usage : + lizard [arg] [input] [output] + +input : a filename + with no FILE, or when FILE is - or stdin, read standard input +Arguments : + -10...-19 : compression method fastLZ4 = 16-bit bytewise codewords + higher number == more compression but slower + -20...-29 : compression method LIZv1 = 24-bit bytewise codewords + -30...-39 : compression method fastLZ4 + Huffman + -40...-49 : compression method LIZv1 + Huffman + -d : decompression (default for .liz extension) + -z : force compression + -f : overwrite output without prompting +--rm : remove source file(s) after successful de/compression + -h/-H : display help/long help and exit + +Advanced arguments : + -V : display Version number and exit + -v : verbose mode + -q : suppress warnings; specify twice to suppress errors too + -c : force write to standard output, even if it is the console + -t : test compressed file integrity + -m : multiple input files (implies automatic output filenames) + -r : operate recursively on directories (sets also -m) + -l : compress using Legacy format (Linux kernel compression) + -B# : Block size [1-7] = 128KB, 256KB, 1MB, 4MB, 16MB, 64MB, 256MB (default : 4) + -BD : Block dependency (improve compression ratio) +--no-frame-crc : disable stream checksum (default:enabled) +--content-size : compressed frame includes original size (default:not present) +--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout) +Benchmark arguments : + -b# : benchmark file(s), using # compression level (default : 1) + -e# : test all compression levels from -bX to # (default : 1) + -i# : minimum evaluation time in seconds (default : 3s) + -B# : cut file into independent blocks of size # bytes [32+] + or predefined block size [1-7] (default: 4) +``` + +#### License + +All files in this directory are licensed under GPL-v2. +See [COPYING](COPYING) for details. +The text of the license is also included at the top of each source file. diff --git a/contrib/lizard/programs/bench.c b/contrib/lizard/programs/bench.c new file mode 100644 index 00000000000..3fe56d1c060 --- /dev/null +++ b/contrib/lizard/programs/bench.c @@ -0,0 +1,502 @@ +/* + bench.c - Demo program to benchmark open-source compression algorithms + Copyright (C) Yann Collet 2012-2016 + Copyright (C) Przemyslaw Skibinski 2016-2017 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + + +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif +#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) +# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ +#endif + + +/* ************************************* +* Includes +***************************************/ +#include "platform.h" /* Compiler options */ +#include /* malloc, free */ +#include /* memset */ +#include /* fprintf, fopen, ftello64 */ +#include /* clock_t, clock, CLOCKS_PER_SEC */ + +#include "datagen.h" /* RDG_genBuffer */ +#include "xxhash/xxhash.h" +#include "lizard_common.h" +#include "lizard_decompress.h" +#define UTIL_WITHOUT_BASIC_TYPES +#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */ + +#define Lizard_isError(errcode) (errcode==0) + + +/* ************************************* +* Constants +***************************************/ +#ifndef LIZARD_GIT_COMMIT_STRING +# define LIZARD_GIT_COMMIT_STRING "" +#else +# define LIZARD_GIT_COMMIT_STRING LIZARD_EXPAND_AND_QUOTE(Lizard_GIT_COMMIT) +#endif + +#define NBSECONDS 3 +#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ +#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ +#define COOLPERIOD_SEC 10 +#define DECOMP_MULT 2 /* test decompression DECOMP_MULT times longer than compression */ + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); + +static U32 g_compressibilityDefault = 50; + + +/* ************************************* +* console display +***************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ + { g_time = clock(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stdout); } } +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; + + +/* ************************************* +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ + exit(error); \ +} + + +/* ************************************* +* Benchmark Parameters +***************************************/ +static U32 g_nbSeconds = NBSECONDS; +static size_t g_blockSize = 0; +int g_additionalParam = 0; + +void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } + +void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } + +void BMK_SetNbSeconds(unsigned nbSeconds) +{ + g_nbSeconds = nbSeconds; + DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds); +} + +void BMK_SetBlockSize(size_t blockSize) +{ + g_blockSize = blockSize; +} + + +/* ******************************************************** +* Bench functions +**********************************************************/ +typedef struct { + const char* srcPtr; + size_t srcSize; + char* cPtr; + size_t cRoom; + size_t cSize; + char* resPtr; + size_t resSize; +} blockParam_t; + + + +static int BMK_benchMem(const void* srcBuffer, size_t srcSize, + const char* displayName, int cLevel, + const size_t* fileSizes, U32 nbFiles) +{ + size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); + size_t const maxCompressedSize = Lizard_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ + void* const compressedBuffer = malloc(maxCompressedSize); + void* const resultBuffer = malloc(srcSize); + U32 nbBlocks; + UTIL_time_t ticksPerSecond; + + /* checks */ + if (!compressedBuffer || !resultBuffer || !blockTable) + EXM_THROW(31, "allocation error : not enough memory"); + + /* init */ + if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ + UTIL_initTimer(&ticksPerSecond); + + /* Init blockTable data */ + { const char* srcPtr = (const char*)srcBuffer; + char* cPtr = (char*)compressedBuffer; + char* resPtr = (char*)resultBuffer; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb ACTIVEPERIOD_MICROSEC) { + DISPLAYLEVEL(2, "\rcooling down ... \r"); + UTIL_sleep(COOLPERIOD_SEC); + UTIL_getTime(&coolTime); + } + + /* Compression */ + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); + if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ + + UTIL_sleepMilli(1); /* give processor time to other processes */ + UTIL_waitForNextTick(ticksPerSecond); + UTIL_getTime(&clockStart); + + if (!cCompleted) { /* still some time to do compression tests */ + U32 nbLoops = 0; + do { + U32 blockNb; + for (blockNb=0; blockNbmaxTime; + } } + + cSize = 0; + { U32 blockNb; for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s\r", + marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, + (double)srcSize / fastestC ); + + (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ +#if 1 + /* Decompression */ + if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ + + UTIL_sleepMilli(1); /* give processor time to other processes */ + UTIL_waitForNextTick(ticksPerSecond); + UTIL_getTime(&clockStart); + + if (!dCompleted) { + U32 nbLoops = 0; + do { + U32 blockNb; + for (blockNb=0; blockNb(DECOMP_MULT*maxTime); + } } + + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", + marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, + (double)srcSize / fastestC, + (double)srcSize / fastestD ); + + /* CRC Checking */ + { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + if (crcOrig!=crcCheck) { + size_t u; + DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); + for (u=0; u u) break; + bacc += blockTable[segNb].srcSize; + } + pos = (U32)(u - bacc); + bNb = pos / (128 KB); + DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos); + break; + } + if (u==srcSize-1) { /* should never happen */ + DISPLAY("no difference detected\n"); + } } + break; + } } /* CRC Checking */ +#endif + } /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */ + + if (g_displayLevel == 1) { + double cSpeed = (double)srcSize / fastestC; + double dSpeed = (double)srcSize / fastestD; + if (g_additionalParam) + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam); + else + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); + } + DISPLAYLEVEL(2, "%2i#\n", cLevel); + } /* Bench */ + + /* clean up */ + free(blockTable); + free(compressedBuffer); + free(resultBuffer); + return 0; +} + + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t step = 64 MB; + BYTE* testmem=NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += 2*step; + if (requiredMem > maxMemory) requiredMem = maxMemory; + + while (!testmem) { + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + testmem = (BYTE*) malloc ((size_t)requiredMem); + } + free (testmem); + + /* keep some space available */ + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + + return (size_t)requiredMem; +} + + +static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, + const char* displayName, int cLevel, int cLevelLast, + const size_t* fileSizes, unsigned nbFiles) +{ + int l; + + const char* pch = strrchr(displayName, '\\'); /* Windows */ + if (!pch) pch = strrchr(displayName, '/'); /* Linux */ + if (pch) displayName = pch+1; + + SET_REALTIME_PRIORITY; + + if (g_displayLevel == 1 && !g_additionalParam) + DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", LIZARD_VERSION_STRING, LIZARD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); + + if (cLevelLast < cLevel) cLevelLast = cLevel; + + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles); + } +} + + +/*! BMK_loadFiles() : + Loads `buffer` with content of files listed within `fileNamesTable`. + At most, fills `buffer` entirely */ +static void BMK_loadFiles(void* buffer, size_t bufferSize, + size_t* fileSizes, + const char** fileNamesTable, unsigned nbFiles) +{ + size_t pos = 0, totalSize = 0; + unsigned n; + for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); + DISPLAYLEVEL(4, "readSize=%llu\n", (unsigned long long)readSize); + pos += readSize; } + fileSizes[n] = (size_t)fileSize; + totalSize += (size_t)fileSize; + fclose(f); + } + + if (totalSize == 0) EXM_THROW(12, "no data to bench"); +} + +static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, + int cLevel, int cLevelLast) +{ + void* srcBuffer; + size_t benchedSize; + size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + char mfName[20] = {0}; + + if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); + + /* Memory allocation & restrictions */ + benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + if (benchedSize==0) EXM_THROW(12, "not enough memory"); + if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; + if (benchedSize > LIZARD_MAX_INPUT_SIZE) { + benchedSize = LIZARD_MAX_INPUT_SIZE; + DISPLAY("File(s) bigger than Lizard's max input size; testing %u MB only...\n", (U32)(benchedSize >> 20)); + } + else + if (benchedSize < totalSizeToLoad) { + DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); + } + srcBuffer = malloc(benchedSize + !benchedSize); /* avoid alloc of zero */ + if (!srcBuffer) EXM_THROW(12, "not enough memory"); + + /* Load input buffer */ + BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); + + /* Bench */ + snprintf (mfName, sizeof(mfName), " %u files", nbFiles); + { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; + BMK_benchCLevel(srcBuffer, benchedSize, + displayName, cLevel, cLevelLast, + fileSizes, nbFiles); + } + + /* clean up */ + free(srcBuffer); + free(fileSizes); +} + + +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) +{ + char name[20] = {0}; + size_t benchedSize = 10000000; + void* const srcBuffer = malloc(benchedSize); + + /* Memory allocation */ + if (!srcBuffer) EXM_THROW(21, "not enough memory"); + + /* Fill input buffer */ + RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); + + /* Bench */ + snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1); + + /* clean up */ + free(srcBuffer); +} + + +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, + int cLevel, int cLevelLast) +{ + double const compressibility = (double)g_compressibilityDefault / 100; + + if (cLevel < LIZARD_MIN_CLEVEL) cLevel = LIZARD_MIN_CLEVEL; + if (cLevel > LIZARD_MAX_CLEVEL) cLevel = LIZARD_MAX_CLEVEL; + if (cLevelLast > LIZARD_MAX_CLEVEL) cLevelLast = LIZARD_MAX_CLEVEL; + if (cLevelLast < cLevel) cLevelLast = cLevel; + if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); + + if (nbFiles == 0) + BMK_syntheticTest(cLevel, cLevelLast, compressibility); + else + BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast); + return 0; +} diff --git a/contrib/lizard/programs/bench.h b/contrib/lizard/programs/bench.h new file mode 100644 index 00000000000..feff51293f3 --- /dev/null +++ b/contrib/lizard/programs/bench.h @@ -0,0 +1,37 @@ +/* + bench.h - Demo program to benchmark open-source compression algorithm + Copyright (C) Yann Collet 2012-2016 + Copyright (C) Przemyslaw Skibinski 2016-2017 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ +#ifndef BENCH_H_12 +#define BENCH_H_12 + +#include + +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, + int cLevel, int cLevelLast); + +/* Set Parameters */ +void BMK_SetNbSeconds(unsigned nbLoops); +void BMK_SetBlockSize(size_t blockSize); +void BMK_setAdditionalParam(int additionalParam); +void BMK_setNotificationLevel(unsigned level); + +#endif /* BENCH_H_125623623633 */ diff --git a/contrib/lizard/programs/datagen.c b/contrib/lizard/programs/datagen.c new file mode 100644 index 00000000000..87d6044830a --- /dev/null +++ b/contrib/lizard/programs/datagen.c @@ -0,0 +1,188 @@ +/* + datagen.c - compressible data generator test tool + Copyright (C) Yann Collet 2012-2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + +/************************************** +* Includes +**************************************/ +#include "platform.h" /* Compiler options, SET_BINARY_MODE */ +#include "util.h" /* U32 */ +#include /* malloc */ +#include /* FILE, fwrite */ +#include /* memcpy */ + + +/************************************** +* Constants +**************************************/ +#define KB *(1 <<10) + +#define PRIME1 2654435761U +#define PRIME2 2246822519U + + +/************************************** +* Local types +**************************************/ +#define LTLOG 13 +#define LTSIZE (1<> (32 - r))) +static unsigned int RDG_rand(U32* src) +{ + U32 rand32 = *src; + rand32 *= PRIME1; + rand32 ^= PRIME2; + rand32 = RDG_rotl32(rand32, 13); + *src = rand32; + return rand32; +} + + +static void RDG_fillLiteralDistrib(litDistribTable lt, double ld) +{ + BYTE const firstChar = ld <= 0.0 ? 0 : '('; + BYTE const lastChar = ld <= 0.0 ? 255 : '}'; + BYTE character = ld <= 0.0 ? 0 : '0'; + U32 u = 0; + + while (u lastChar) character = firstChar; + } +} + + +static BYTE RDG_genChar(U32* seed, const litDistribTable lt) +{ + U32 id = RDG_rand(seed) & LTMASK; + return (lt[id]); +} + + +#define RDG_DICTSIZE (32 KB) +#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767) +#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15) +void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr) +{ + BYTE* buffPtr = (BYTE*)buffer; + const U32 matchProba32 = (U32)(32768 * matchProba); + size_t pos = prefixSize; + U32* seed = seedPtr; + + /* special case */ + while (matchProba >= 1.0) + { + size_t size0 = RDG_rand(seed) & 3; + size0 = (size_t)1 << (16 + size0 * 2); + size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/ + if (buffSize < pos + size0) + { + memset(buffPtr+pos, 0, buffSize-pos); + return; + } + memset(buffPtr+pos, 0, size0); + pos += size0; + buffPtr[pos-1] = RDG_genChar(seed, lt); + } + + /* init */ + if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1; + + /* Generate compressible data */ + while (pos < buffSize) + { + /* Select : Literal (char) or Match (within 32K) */ + if (RDG_RAND15BITS < matchProba32) + { + /* Copy (within 32K) */ + size_t match; + size_t d; + int length = RDG_RANDLENGTH + 4; + U32 offset = RDG_RAND15BITS + 1; + if (offset > pos) offset = (U32)pos; + match = pos - offset; + d = pos + length; + if (d > buffSize) d = buffSize; + while (pos < d) buffPtr[pos++] = buffPtr[match++]; + } + else + { + /* Literal (noise) */ + size_t d; + size_t length = RDG_RANDLENGTH; + d = pos + length; + if (d > buffSize) d = buffSize; + while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt); + } + } +} + + +void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) +{ + litDistribTable lt; + if (litProba==0.0) litProba = matchProba / 4.5; + RDG_fillLiteralDistrib(lt, litProba); + RDG_genBlock(buffer, size, 0, matchProba, lt, &seed); +} + + +#define RDG_BLOCKSIZE (128 KB) +void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed) +{ + BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE]; + U64 total = 0; + size_t genBlockSize = RDG_BLOCKSIZE; + litDistribTable lt; + + /* init */ + if (litProba==0.0) litProba = matchProba / 4.5; + RDG_fillLiteralDistrib(lt, litProba); + SET_BINARY_MODE(stdout); + + /* Generate dict */ + RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, lt, &seed); + + /* Generate compressible data */ + while (total < size) + { + RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, lt, &seed); + if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total); + total += genBlockSize; + fwrite(buff, 1, genBlockSize, stdout); + /* update dict */ + memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE); + } +} diff --git a/contrib/lizard/programs/datagen.h b/contrib/lizard/programs/datagen.h new file mode 100644 index 00000000000..2cc41d2a3ff --- /dev/null +++ b/contrib/lizard/programs/datagen.h @@ -0,0 +1,39 @@ +/* + datagen.h - compressible data generator header + Copyright (C) Yann Collet 2012-2015 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + + +#include /* size_t */ + +void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed); +void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed); +/* RDG_genOut + Generate 'size' bytes of compressible data into stdout. + Compressibility can be controlled using 'matchProba'. + 'LitProba' is optional, and affect variability of bytes. If litProba==0.0, default value is used. + Generated data can be selected using 'seed'. + If (matchProba, litProba and seed) are equal, the function always generate the same content. + + RDG_genBuffer + Same as RDG_genOut, but generate data into provided buffer +*/ diff --git a/contrib/lizard/programs/lizard.1 b/contrib/lizard/programs/lizard.1 new file mode 100644 index 00000000000..b5d84213442 --- /dev/null +++ b/contrib/lizard/programs/lizard.1 @@ -0,0 +1,229 @@ +\." +\." lizard.1: This is a manual page for 'lizard' program. This file is part of the +\." lizard project. +\." Author: Yann Collet +\." +. +\." No hyphenation +.hy 0 +.nr HY 0 +. +.TH lizard "1" "2015-03-21" "lizard" "User Commands" +.SH NAME +\fBlizard, unlizard, lizardcat\fR \- Compress or decompress .liz files + +.SH SYNOPSIS +.TP 5 +\fBlizard\fR [\fBOPTIONS\fR] [-|INPUT-FILE] +.PP +.B unlizard +is equivalent to +.BR "lizard \-d" +.br +.B lizardcat +is equivalent to +.BR "lizard \-dc" +.br +.PP +When writing scripts that need to decompress files, +it is recommended to always use the name +.B lizard +with appropriate arguments +.RB ( "lizard \-d" +or +.BR "lizard \-dc" ) +instead of the names +.B unlizard +and +.BR lizardcat . + + +.SH DESCRIPTION +.PP +\fBlizard\fR is an extremely fast lossless compression algorithm, +based on \fBbyte-aligned LZ77\fR family of compression scheme. +\fBlizard\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi-core CPUs. +It features an extremely fast decoder, with speed in multiple GB/s per core, +typically reaching RAM speed limit on multi-core systems. +The native file format is the +.B .liz +format. + +.B lizard +supports a command line syntax similar but not identical to +.BR gzip (1). +Differences are : +\fBlizard\fR preserve original files ; +\fBlizard file1 file2\fR means : compress file1 \fIinto\fR file2 ; +\fBlizard file\fR shows real-time statistics during compression . + +Default behaviors can be modified by opt-in commands, described below. +\fBlizard --quiet --multiple\fR more closely mimics \fBgzip\fR behavior. + +.SS "Concatenation of .liz files" +It is possible to concatenate +.B .liz +files as is. +.B lizard +will decompress such files as if they were a single +.B .liz +file. For example: + lizard file1 > foo.liz + lizard file2 >> foo.liz + then + lizardcat foo.liz + is equivalent to : + cat file1 file2 + +.PP + +.SH OPTIONS +. +.SS "Short commands concatenation" +In some cases, some options can be expressed using short command +.B "-x" +or long command +.B "--long-word" . +Short commands can be concatenated together. For example, +.B "-d -c" +is equivalent to +.B "-dc" . +Long commands cannot be concatenated. +They must be clearly separated by a space. +.SS "Multiple commands" +When multiple contradictory commands are issued on a same command line, +only the latest one will be applied. +. +.SS "Operation mode" +.TP +.BR \-z ", " \-\-compress +Compress. +This is the default operation mode +when no operation mode option is specified , +no other operation mode is implied from the command name +(for example, +.B unlizard +implies +.B \-\-decompress ), +nor from the input file name +(for example, a file extension +.B .liz +implies +.B \-\-decompress +by default). +.B -z +can also be used to force compression of an already compressed +.B .liz +file. +.TP +.BR \-d ", " \-\-decompress ", " \-\-uncompress +Decompress. +.B --decompress +is also the default operation when the input filename has an +.B .liz +extensionq +.TP +.BR \-t ", " \-\-test +Test the integrity of compressed +.B .liz +files. +The decompressed data is discarded. +No files are created nor removed. +. +.SS "Operation modifiers" +.TP +.B \-1 + fast compression (default) +.TP +.B \-9 + high compression + +.TP +.BR \-f ", " --[no-]force + This option has several effects: +.RS +.IP \(bu 3 +If the target file already exists, +overwrite it without prompting. +.IP \(bu 3 +When used with +.B \-\-decompress +and +.B lizard +cannot recognize the type of the source file, +copy the source file as is to standard output. +This allows +.B lizardcat +.B \-\-force +to be used like +.BR cat (1) +for files that have not been compressed with +.BR lizard . +.RE + +.TP +.BR \-c ", " \--stdout ", " \--to-stdout + force write to standard output, even if it is the console + +.TP +.BR \-m ", " \--multiple + Multiple file names. + By default, the second filename is used as the destination filename for the compressed file. + With +.B -m +, you can specify any number of input filenames. Each of them will be compressed +independently, and the resulting name of each compressed file will be +.B filename.liz +. + +.TP +.B \-B# + block size [4-7](default : 7) + B1 = 128KB, B2=256KB, B3=1MB, B4=4MB, B5=16MB, B6=64MB, B7=256MB +.TP +.B \-BD + block dependency (improves compression ratio on small blocks) +.TP +.B \--[no-]frame-crc + select frame checksum (default:enabled) +.TP +.B \--[no-]content-size + header includes original size (default:not present) + Note : this option can only be activated when the original size can be determined, +hence for a file. It won't work with unknown source size, such as stdin or pipe. +.TP +.B \--[no-]sparse + sparse mode support (default:enabled on file, disabled on stdout) +.TP +.B \-l + use Legacy format (useful for Linux Kernel compression) +. +.SS "Other options" +.TP +.BR \-v ", " --verbose + verbose mode +.TP +.BR \-q ", " --quiet + suppress warnings and real-time statistics; specify twice to suppress errors too +.TP +.B \-h/\-H + display help/long help and exit +.TP +.BR \-V ", " \--version + display Version number and exit +.TP +.BR \-k ", " \--keep + Don't delete source file. +This is default behavior anyway, so this option is just for compatibility with gzip/xz. +.TP +.B \-b + benchmark file(s) +.TP +.B \-i# + iteration loops [1-9](default : 3), benchmark mode only + +.SH BUGS +Report bugs at: https://github.com/inikep/lizard/issues + +.SH AUTHOR +Yann Collet diff --git a/contrib/lizard/programs/lizardcli.c b/contrib/lizard/programs/lizardcli.c new file mode 100644 index 00000000000..2a63fe8e001 --- /dev/null +++ b/contrib/lizard/programs/lizardcli.c @@ -0,0 +1,581 @@ +/* + Lizardcli - Lizard Command Line Interface + Copyright (C) Yann Collet 2011-2016 + Copyright (C) Przemyslaw Skibinski 2016-2017 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ +/* + Note : this is stand-alone program. + It is not part of Lizard compression library, it is a user program of the Lizard library. + The license of Lizard library is BSD. + The license of xxHash library is BSD. + The license of this compression CLI program is GPLv2. +*/ + + +/**************************** +* Includes +*****************************/ +#include "platform.h" /* Compiler options, IS_CONSOLE */ +#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ +#include /* fprintf, getchar */ +#include /* exit, calloc, free */ +#include /* strcmp, strlen */ +#include "bench.h" /* BMK_benchFile, BMK_SetNbIterations, BMK_SetBlocksize, BMK_SetPause */ +#include "lizardio.h" /* LizardIO_compressFilename, LizardIO_decompressFilename, LizardIO_compressMultipleFilenames */ +#include "lizard_compress.h" /* LizardHC_DEFAULT_CLEVEL, LIZARD_VERSION_STRING */ + + + +/***************************** +* Constants +******************************/ +#define COMPRESSOR_NAME "Lizard command line interface" +#define AUTHOR "Y.Collet & P.Skibinski" +#define WELCOME_MESSAGE "%s %i-bit %s by %s (%s)\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LIZARD_VERSION_STRING, AUTHOR, __DATE__ +#define LIZARD_EXTENSION ".liz" +#define LIZARDCAT "lizardcat" +#define UNLIZARD "unlizard" + +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +#define LIZARD_BLOCKSIZEID_DEFAULT 4 + + +/*-************************************ +* Macros +***************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */ + + +/*-************************************ +* Exceptions +***************************************/ +#define DEBUG 0 +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ + exit(error); \ +} + + +/*-************************************ +* Version modifiers +***************************************/ +#define EXTENDED_ARGUMENTS +#define EXTENDED_HELP +#define EXTENDED_FORMAT +#define DEFAULT_COMPRESSOR LizardIO_compressFilename +#define DEFAULT_DECOMPRESSOR LizardIO_decompressFilename +int LizardIO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */ + + +/*-*************************** +* Functions +*****************************/ +static int usage(const char* exeName) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] [input] [output]\n", exeName); + DISPLAY( "\n"); + DISPLAY( "input : a filename\n"); + DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); + DISPLAY( "Arguments :\n"); + DISPLAY( " -10...-19 : compression method fastLZ4 = 16-bit bytewise codewords\n"); + DISPLAY( " higher number == more compression but slower\n"); + DISPLAY( " -20...-29 : compression method LIZv1 = 24-bit bytewise codewords\n"); +#ifndef LIZARD_NO_HUFFMAN + DISPLAY( " -30...-39 : compression method fastLZ4 + Huffman\n"); + DISPLAY( " -40...-49 : compression method LIZv1 + Huffman\n"); +#endif + DISPLAY( " -d : decompression (default for %s extension)\n", LIZARD_EXTENSION); + DISPLAY( " -z : force compression\n"); + DISPLAY( " -f : overwrite output without prompting \n"); + DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); + DISPLAY( " -h/-H : display help/long help and exit\n"); + return 0; +} + +static int usage_advanced(const char* exeName) +{ + DISPLAY(WELCOME_MESSAGE); + usage(exeName); + DISPLAY( "\n"); + DISPLAY( "Advanced arguments :\n"); + DISPLAY( " -V : display Version number and exit\n"); + DISPLAY( " -v : verbose mode\n"); + DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); + DISPLAY( " -c : force write to standard output, even if it is the console\n"); + DISPLAY( " -t : test compressed file integrity\n"); + DISPLAY( " -m : multiple input files (implies automatic output filenames)\n"); +#ifdef UTIL_HAS_CREATEFILELIST + DISPLAY( " -r : operate recursively on directories (sets also -m)\n"); +#endif + DISPLAY( " -B# : Block size [1-7] = 128KB, 256KB, 1MB, 4MB, 16MB, 64MB, 256MB (default : 4)\n"); + DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); + /* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */ + DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n"); + DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); + DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); + DISPLAY( "Benchmark arguments :\n"); + DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); + DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n"); + DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); + DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+]\n"); + DISPLAY( " or predefined block size [1-7] (default: 4)\n"); + EXTENDED_HELP; + return 0; +} + +static int usage_longhelp(const char* exeName) +{ + usage_advanced(exeName); + DISPLAY( "\n"); + DISPLAY( "****************************\n"); + DISPLAY( "***** Advanced comment *****\n"); + DISPLAY( "****************************\n"); + DISPLAY( "\n"); + DISPLAY( "Which values can [output] have ? \n"); + DISPLAY( "---------------------------------\n"); + DISPLAY( "[output] : a filename \n"); + DISPLAY( " '%s', or '-' for standard output (pipe mode)\n", stdoutmark); + DISPLAY( " '%s' to discard output (test mode) \n", NULL_OUTPUT); + DISPLAY( "[output] can be left empty. In this case, it receives the following value :\n"); + DISPLAY( " - if stdout is not the console, then [output] = stdout \n"); + DISPLAY( " - if stdout is console : \n"); + DISPLAY( " + for compression, output to filename%s \n", LIZARD_EXTENSION); + DISPLAY( " + for decompression, output to filename without '%s'\n", LIZARD_EXTENSION); + DISPLAY( " > if input filename has no '%s' extension : error \n", LIZARD_EXTENSION); + DISPLAY( "\n"); + DISPLAY( "stdin, stdout and the console : \n"); + DISPLAY( "--------------------------------\n"); + DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n"); + DISPLAY( "%s will refuse to read from console, or write to console \n", exeName); + DISPLAY( "except if '-c' command is specified, to force output to console \n"); + DISPLAY( "\n"); + DISPLAY( "Simple example :\n"); + DISPLAY( "----------------\n"); + DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.liz'\n"); + DISPLAY( " %s filename\n", exeName); + DISPLAY( "\n"); + DISPLAY( "Short arguments can be aggregated. For example :\n"); + DISPLAY( "----------------------------------\n"); + DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n"); + DISPLAY( " %s -9 -f filename \n", exeName); + DISPLAY( " is equivalent to :\n"); + DISPLAY( " %s -9f filename \n", exeName); + DISPLAY( "\n"); + DISPLAY( "%s can be used in 'pure pipe mode'. For example :\n", exeName); + DISPLAY( "-------------------------------------\n"); + DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n"); + DISPLAY( " generator | %s | consumer \n", exeName); + return 0; +} + +static int badusage(const char* exeName) +{ + DISPLAYLEVEL(1, "Incorrect parameters\n"); + if (displayLevel >= 1) usage(exeName); + exit(1); +} + + +static void waitEnter(void) +{ + DISPLAY("Press enter to continue...\n"); + (void)getchar(); +} + + +static const char* lastNameFromPath(const char* path) +{ + const char* name = strrchr(path, '/'); + if (name==NULL) name = strrchr(path, '\\'); /* windows */ + if (name==NULL) return path; + return name+1; +} + + +/*! readU32FromChar() : + @return : unsigned integer value reach from input in `char` format + Will also modify `*stringPtr`, advancing it to position where it stopped reading. + Note : this function can overflow if result > MAX_UINT */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + return result; +} + +typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench } operationMode_e; + +int main(int argc, const char** argv) +{ + int i, + cLevel=1, + cLevelLast=1, + forceStdout=0, + main_pause=0, + multiple_inputs=0, + operationResult=0; + operationMode_e mode = om_auto; + const char* input_filename = NULL; + const char* output_filename= NULL; + char* dynNameSpace = NULL; + const char** inFileNames = (const char**) calloc(argc, sizeof(char*)); + unsigned ifnIdx=0; + const char nullOutput[] = NULL_OUTPUT; + const char extension[] = LIZARD_EXTENSION; + size_t blockSize = LizardIO_setBlockSizeID(LIZARD_BLOCKSIZEID_DEFAULT); + const char* const exeName = lastNameFromPath(argv[0]); +#ifdef UTIL_HAS_CREATEFILELIST + const char** extendedFileList = NULL; + char* fileNamesBuf = NULL; + unsigned fileNamesNb, recursive=0; +#endif + + /* Init */ + if (inFileNames==NULL) { + DISPLAY("Allocation error : not enough memory \n"); + return 1; + } + inFileNames[0] = stdinmark; + LizardIO_setOverwrite(0); + + /* lizardcat predefined behavior */ + if (!strcmp(exeName, LIZARDCAT)) { + mode = om_decompress; + LizardIO_setOverwrite(1); + forceStdout=1; + output_filename=stdoutmark; + displayLevel=1; + multiple_inputs=1; + } + if (!strcmp(exeName, UNLIZARD)) { mode = om_decompress; } + + /* command switches */ + for(i=1; i='0') && (*argument<='9')) { + cLevel = readU32FromChar(&argument); + argument--; + continue; + } + + + switch(argument[0]) + { + /* Display help */ + case 'V': DISPLAY(WELCOME_MESSAGE); goto _cleanup; /* Version */ + case 'h': usage_advanced(exeName); goto _cleanup; + case 'H': usage_longhelp(exeName); goto _cleanup; + + case 'e': + argument++; + cLevelLast = readU32FromChar(&argument); + argument--; + break; + + /* Compression (default) */ + case 'z': mode = om_compress; break; + + /* Decoding */ + case 'd': mode = om_decompress; break; + + /* Force stdout, even if stdout==console */ + case 'c': forceStdout=1; output_filename=stdoutmark; break; + + /* Test integrity */ + case 't': mode = om_test; break; + + /* Overwrite */ + case 'f': LizardIO_setOverwrite(1); break; + + /* Verbose mode */ + case 'v': displayLevel++; break; + + /* Quiet mode */ + case 'q': if (displayLevel) displayLevel--; break; + + /* keep source file (default anyway, so useless) (for xz/lzma compatibility) */ + case 'k': LizardIO_setRemoveSrcFile(0); break; + + /* Modify Block Properties */ + case 'B': + while (argument[1]!=0) { + int exitBlockProperties=0; + switch(argument[1]) + { + case 'D': LizardIO_setBlockMode(LizardIO_blockLinked); argument++; break; + case 'X': LizardIO_setBlockChecksumMode(1); argument ++; break; /* disabled by default */ + default : + if (argument[1] < '0' || argument[1] > '9') { + exitBlockProperties=1; + break; + } else { + unsigned B; + argument++; + B = readU32FromChar(&argument); + argument--; + if (B < 1) badusage(exeName); + if (B <= 7) { + blockSize = LizardIO_setBlockSizeID(B); + BMK_SetBlockSize(blockSize); + DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); + } else { + if (B < 32) badusage(exeName); + BMK_SetBlockSize(B); + if (B >= 1024) { + DISPLAYLEVEL(2, "bench: using blocks of size %u KB \n", (U32)(B>>10)); + } else { + DISPLAYLEVEL(2, "bench: using blocks of size %u bytes \n", (U32)(B)); + } + } + break; + } + } + if (exitBlockProperties) break; + } + break; + + /* Benchmark */ + case 'b': mode = om_bench; multiple_inputs=1; + break; + +#ifdef UTIL_HAS_CREATEFILELIST + /* recursive */ + case 'r': recursive=1; + /* fallthrough */ +#endif + /* Treat non-option args as input files. See https://code.google.com/p/lizard/issues/detail?id=151 */ + /* fallthrough */ + case 'm': multiple_inputs=1; + break; + + /* Modify Nb Seconds (benchmark only) */ + case 'i': + { unsigned iters; + argument++; + iters = readU32FromChar(&argument); + argument--; + BMK_setNotificationLevel(displayLevel); + BMK_SetNbSeconds(iters); /* notification if displayLevel >= 3 */ + } + break; + + /* Pause at the end (hidden option) */ + case 'p': main_pause=1; break; + + /* Specific commands for customized versions */ + EXTENDED_ARGUMENTS; + + /* Unrecognised command */ + default : badusage(exeName); + } + } + continue; + } + + /* Store in *inFileNames[] if -m is used. */ + if (multiple_inputs) { inFileNames[ifnIdx++]=argument; continue; } + + /* Store first non-option arg in input_filename to preserve original cli logic. */ + if (!input_filename) { input_filename=argument; continue; } + + /* Second non-option arg in output_filename to preserve original cli logic. */ + if (!output_filename) { + output_filename=argument; + if (!strcmp (output_filename, nullOutput)) output_filename = nulmark; + continue; + } + + /* 3rd non-option arg should not exist */ + DISPLAYLEVEL(1, "Warning : %s won't be used ! Do you want multiple input files (-m) ? \n", argument); + } + + DISPLAYLEVEL(3, WELCOME_MESSAGE); + if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %i KB\n", (U32)(blockSize>>10)); + + if (multiple_inputs) { + input_filename = inFileNames[0]; +#ifdef UTIL_HAS_CREATEFILELIST + if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ + extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb); + if (extendedFileList) { + unsigned u; + for (u=0; u try to select one automatically (when possible) */ + while ((!output_filename) && (multiple_inputs==0)) { + if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ + if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */ + size_t const inSize = strlen(input_filename); + size_t const extSize = strlen(LIZARD_EXTENSION); + size_t const extStart= (inSize > extSize) ? inSize-extSize : 0; + if (!strcmp(input_filename+extStart, LIZARD_EXTENSION)) mode = om_decompress; + else mode = om_compress; + } + if (mode == om_compress) { /* compression to file */ + size_t const l = strlen(input_filename); + dynNameSpace = (char*)calloc(1,l+5); + if (dynNameSpace==NULL) { perror(exeName); exit(1); } + strcpy(dynNameSpace, input_filename); + strcat(dynNameSpace, LIZARD_EXTENSION); + output_filename = dynNameSpace; + DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename); + break; + } + if (mode == om_decompress) {/* decompression to file (automatic name will work only if input filename has correct format extension) */ + size_t outl; + size_t const inl = strlen(input_filename); + dynNameSpace = (char*)calloc(1,inl+1); + if (dynNameSpace==NULL) { perror(exeName); exit(1); } + strcpy(dynNameSpace, input_filename); + outl = inl; + if (inl>4) + while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) dynNameSpace[outl--]=0; + if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(exeName); } + output_filename = dynNameSpace; + DISPLAYLEVEL(2, "Decoding file %s \n", output_filename); + } + break; + } + + /* Check if output is defined as console; trigger an error in this case */ + if (!output_filename) output_filename = "*\\dummy^!//"; + if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) { + DISPLAYLEVEL(1, "refusing to write to console without -c\n"); + exit(1); + } + + /* Downgrade notification level in stdout and multiple file mode */ + if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; + if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; + + /* IO Stream/File */ + LizardIO_setNotificationLevel(displayLevel); + if (ifnIdx == 0) multiple_inputs = 0; + if (mode == om_decompress) { + if (multiple_inputs) + operationResult = LizardIO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LIZARD_EXTENSION); + else + operationResult = LizardIO_decompressFilename(input_filename, output_filename); + } else { /* compression is default action */ + { + if (multiple_inputs) + operationResult = LizardIO_compressMultipleFilenames(inFileNames, ifnIdx, LIZARD_EXTENSION, cLevel); + else + operationResult = LizardIO_compressFilename(input_filename, output_filename, cLevel); + } + } + +_cleanup: + if (main_pause) waitEnter(); + if (dynNameSpace) free(dynNameSpace); +#ifdef UTIL_HAS_CREATEFILELIST + if (extendedFileList) + UTIL_freeFileList(extendedFileList, fileNamesBuf); + else +#endif + free((void*)inFileNames); + return operationResult; +} diff --git a/contrib/lizard/programs/lizardio.c b/contrib/lizard/programs/lizardio.c new file mode 100644 index 00000000000..7638a55178a --- /dev/null +++ b/contrib/lizard/programs/lizardio.c @@ -0,0 +1,896 @@ +/* + Lizardio.c - Lizard File/Stream Interface + Copyright (C) Yann Collet 2011-2015 + Copyright (C) Przemyslaw Skibinski 2016-2017 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ +/* + Note : this is stand-alone program. + It is not part of Lizard compression library, it is a user code of the Lizard library. + - The license of Lizard library is BSD. + - The license of xxHash library is BSD. + - The license of this source file is GPLv2. +*/ + +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif +#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) +# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ +#endif + + +/***************************** +* Includes +*****************************/ +#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE, PLATFORM_POSIX_VERSION, __64BIT__ */ +#include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ +#include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ +#include /* malloc, free */ +#include /* strcmp, strlen */ +#include /* clock */ +#include /* stat64 */ +#include /* stat64 */ +#include "lizardio.h" +#include "lizard_frame.h" + + + +/***************************** +* Constants +*****************************/ +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define _1BIT 0x01 +#define _2BITS 0x03 +#define _3BITS 0x07 +#define _4BITS 0x0F +#define _8BITS 0xFF + +#define MAGICNUMBER_SIZE 4 +#define LIZARDIO_MAGICNUMBER 0x184D2206U +#define LIZARDIO_SKIPPABLE0 0x184D2A50U +#define LIZARDIO_SKIPPABLEMASK 0xFFFFFFF0U + +#define CACHELINE 64 +#define MIN_STREAM_BUFSIZE (192 KB) +#define LIZARDIO_BLOCKSIZEID_DEFAULT 7 + +#define sizeT sizeof(size_t) +#define maskT (sizeT - 1) + + +/************************************** +* Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if (((clock_t)(g_time - clock()) > refreshRate) || (g_displayLevel>=4)) \ + { g_time = clock(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stderr); } } +static const clock_t refreshRate = CLOCKS_PER_SEC / 6; +static clock_t g_time = 0; + + +/************************************** +* Local Parameters +**************************************/ +static int g_overwrite = 1; +static int g_testMode = 0; +static int g_blockSizeId = LIZARDIO_BLOCKSIZEID_DEFAULT; +static int g_blockChecksum = 0; +static int g_streamChecksum = 1; +static int g_blockIndependence = 1; +static int g_sparseFileSupport = 1; +static int g_contentSizeFlag = 0; + + +/************************************** +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, " \n"); \ + exit(error); \ +} + + +/************************************** +* Version modifiers +**************************************/ +#define EXTENDED_ARGUMENTS +#define EXTENDED_HELP +#define EXTENDED_FORMAT +#define DEFAULT_DECOMPRESSOR LizardIO_decompressLizardF + + +/* ************************************************** */ +/* ****************** Parameters ******************** */ +/* ************************************************** */ + +/* Default setting : overwrite = 1; return : overwrite mode (0/1) */ +int LizardIO_setOverwrite(int yes) +{ + g_overwrite = (yes!=0); + return g_overwrite; +} + +/* Default setting : testMode = 0; return : testMode (0/1) */ +int LizardIO_setTestMode(int yes) +{ + g_testMode = (yes!=0); + return g_testMode; +} + +/* blockSizeID : valid values : 1-7 */ +size_t LizardIO_setBlockSizeID(unsigned bsid) +{ + static const int blockSizeTable[] = { 128 KB, 256 KB, 1 MB, 4 MB, 16 MB, 64 MB, 256 MB }; + static const unsigned minBlockSizeID = 1; + static const unsigned maxBlockSizeID = 7; + if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return 0; + + g_blockSizeId = bsid; + return blockSizeTable[g_blockSizeId-minBlockSizeID]; +} + + +static size_t LizardIO_GetBlockSize_FromBlockId(unsigned blockSizeID) +{ + static const size_t blockSizes[7] = { 128 KB, 256 KB, 1 MB, 4 MB, 16 MB, 64 MB, 256 MB }; + + if (blockSizeID == 0) blockSizeID = LIZARDIO_BLOCKSIZEID_DEFAULT; + blockSizeID -= 1; + if (blockSizeID >= 7) blockSizeID = LIZARDIO_BLOCKSIZEID_DEFAULT - 1; + + return blockSizes[blockSizeID]; +} + + + +int LizardIO_setBlockMode(LizardIO_blockMode_t blockMode) +{ + g_blockIndependence = (blockMode == LizardIO_blockIndependent); + return g_blockIndependence; +} + +/* Default setting : no checksum */ +int LizardIO_setBlockChecksumMode(int xxhash) +{ + g_blockChecksum = (xxhash != 0); + return g_blockChecksum; +} + +/* Default setting : checksum enabled */ +int LizardIO_setStreamChecksumMode(int xxhash) +{ + g_streamChecksum = (xxhash != 0); + return g_streamChecksum; +} + +/* Default setting : 0 (no notification) */ +int LizardIO_setNotificationLevel(int level) +{ + g_displayLevel = level; + return g_displayLevel; +} + +/* Default setting : 0 (disabled) */ +int LizardIO_setSparseFile(int enable) +{ + g_sparseFileSupport = (enable!=0); + return g_sparseFileSupport; +} + +/* Default setting : 0 (disabled) */ +int LizardIO_setContentSize(int enable) +{ + g_contentSizeFlag = (enable!=0); + return g_contentSizeFlag; +} + +static U32 g_removeSrcFile = 0; +void LizardIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } + + + +/* ************************************************************************ ** +** ********************** Lizard File / Pipe compression ********************* ** +** ************************************************************************ */ + +static int LizardIO_isSkippableMagicNumber(unsigned int magic) { return (magic & LIZARDIO_SKIPPABLEMASK) == LIZARDIO_SKIPPABLE0; } + + +/** LizardIO_openSrcFile() : + * condition : `dstFileName` must be non-NULL. + * @result : FILE* to `dstFileName`, or NULL if it fails */ +static FILE* LizardIO_openSrcFile(const char* srcFileName) +{ + FILE* f; + + if (!strcmp (srcFileName, stdinmark)) { + DISPLAYLEVEL(4,"Using stdin for input\n"); + f = stdin; + SET_BINARY_MODE(stdin); + } else { + f = fopen(srcFileName, "rb"); + if ( f==NULL ) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno)); + } + + return f; +} + +/** FIO_openDstFile() : + * condition : `dstFileName` must be non-NULL. + * @result : FILE* to `dstFileName`, or NULL if it fails */ +static FILE* LizardIO_openDstFile(const char* dstFileName) +{ + FILE* f; + + if (!strcmp (dstFileName, stdoutmark)) { + DISPLAYLEVEL(4,"Using stdout for output\n"); + f = stdout; + SET_BINARY_MODE(stdout); + if (g_sparseFileSupport==1) { + g_sparseFileSupport = 0; + DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); + } + } else { + if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ + f = fopen( dstFileName, "rb" ); + if (f != NULL) { /* dest exists, prompt for overwrite authorization */ + fclose(f); + if (g_displayLevel <= 1) { /* No interaction possible */ + DISPLAY("%s already exists; not overwritten \n", dstFileName); + return NULL; + } + DISPLAY("%s already exists; do you wish to overwrite (y/N) ? ", dstFileName); + { int ch = getchar(); + if ((ch!='Y') && (ch!='y')) { + DISPLAY(" not overwritten \n"); + return NULL; + } + while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */ + } } } + f = fopen( dstFileName, "wb" ); + if (f==NULL) DISPLAYLEVEL(1, "%s: %s\n", dstFileName, strerror(errno)); + } + + /* sparse file */ + if (f && g_sparseFileSupport) { SET_SPARSE_FILE_MODE(f); } + + return f; +} + + +/* unoptimized version; solves endianess & alignment issues */ +static void LizardIO_writeLE32 (void* p, unsigned value32) +{ + unsigned char* dstPtr = (unsigned char*)p; + dstPtr[0] = (unsigned char)value32; + dstPtr[1] = (unsigned char)(value32 >> 8); + dstPtr[2] = (unsigned char)(value32 >> 16); + dstPtr[3] = (unsigned char)(value32 >> 24); +} + + + +/********************************************* +* Compression using Frame format +*********************************************/ + +typedef struct { + void* srcBuffer; + size_t srcBufferSize; + void* dstBuffer; + size_t dstBufferSize; + LizardF_compressionContext_t ctx; +} cRess_t; + +static cRess_t LizardIO_createCResources(void) +{ + const size_t blockSize = (size_t)LizardIO_GetBlockSize_FromBlockId (g_blockSizeId); + cRess_t ress; + + LizardF_errorCode_t const errorCode = LizardF_createCompressionContext(&(ress.ctx), LIZARDF_VERSION); + if (LizardF_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LizardF context : %s", LizardF_getErrorName(errorCode)); + + /* Allocate Memory */ + ress.srcBuffer = malloc(blockSize); + ress.srcBufferSize = blockSize; + ress.dstBufferSize = LizardF_compressFrameBound(blockSize, NULL); /* cover worst case */ + ress.dstBuffer = malloc(ress.dstBufferSize); + if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); + + return ress; +} + +static void LizardIO_freeCResources(cRess_t ress) +{ + free(ress.srcBuffer); + free(ress.dstBuffer); + { LizardF_errorCode_t const errorCode = LizardF_freeCompressionContext(ress.ctx); + if (LizardF_isError(errorCode)) EXM_THROW(38, "Error : can't free LizardF context resource : %s", LizardF_getErrorName(errorCode)); } +} + +/* + * LizardIO_compressFilename_extRess() + * result : 0 : compression completed correctly + * 1 : missing or pb opening srcFileName + */ +static int LizardIO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) +{ + unsigned long long filesize = 0; + unsigned long long compressedfilesize = 0; + FILE* srcFile; + FILE* dstFile; + void* const srcBuffer = ress.srcBuffer; + void* const dstBuffer = ress.dstBuffer; + const size_t dstBufferSize = ress.dstBufferSize; + const size_t blockSize = (size_t)LizardIO_GetBlockSize_FromBlockId (g_blockSizeId); + size_t readSize; + LizardF_compressionContext_t ctx = ress.ctx; /* just a pointer */ + LizardF_preferences_t prefs; + + /* Init */ + srcFile = LizardIO_openSrcFile(srcFileName); + if (srcFile == NULL) return 1; + dstFile = LizardIO_openDstFile(dstFileName); + if (dstFile == NULL) { fclose(srcFile); return 1; } + memset(&prefs, 0, sizeof(prefs)); + + + /* Set compression parameters */ + prefs.autoFlush = 1; + prefs.compressionLevel = compressionLevel; + prefs.frameInfo.blockMode = (LizardF_blockMode_t)g_blockIndependence; + prefs.frameInfo.blockSizeID = (LizardF_blockSizeID_t)g_blockSizeId; + prefs.frameInfo.contentChecksumFlag = (LizardF_contentChecksum_t)g_streamChecksum; + if (g_contentSizeFlag) { + U64 const fileSize = UTIL_getFileSize(srcFileName); + prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ + if (fileSize==0) + DISPLAYLEVEL(3, "Warning : cannot determine input content size \n"); + } + + /* read first block */ + readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile); + if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName); + filesize += readSize; + + /* single-block file */ + if (readSize < blockSize) { + /* Compress in single pass */ + size_t const cSize = LizardF_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs); + if (LizardF_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LizardF_getErrorName(cSize)); + compressedfilesize = cSize; + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", + (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */ + + /* Write Block */ + { size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile); + if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block"); + } } + + else + + /* multiple-blocks file */ + { + /* Write Archive Header */ + size_t headerSize = LizardF_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs); + if (LizardF_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LizardF_getErrorName(headerSize)); + { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); + if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); } + compressedfilesize += headerSize; + + /* Main Loop */ + while (readSize>0) { + size_t outSize; + + /* Compress Block */ + outSize = LizardF_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); + if (LizardF_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LizardF_getErrorName(outSize)); + compressedfilesize += outSize; + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); + + /* Write Block */ + { size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); + if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); } + + /* Read next block */ + readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); + filesize += readSize; + } + if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); + + /* End of Stream mark */ + headerSize = LizardF_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); + if (LizardF_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LizardF_getErrorName(headerSize)); + + { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); + if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); } + compressedfilesize += headerSize; + } + + /* Release files */ + fclose (srcFile); + fclose (dstFile); + + /* Copy owner, file permissions and modification time */ + { stat_t statbuf; + if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && UTIL_getFileStat(srcFileName, &statbuf)) + UTIL_setFileStat(dstFileName, &statbuf); + } + + if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ + + /* Final Status */ + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + filesize, compressedfilesize, (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */ + + return 0; +} + + +int LizardIO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel) +{ + clock_t const start = clock(); + cRess_t const ress = LizardIO_createCResources(); + + int const issueWithSrcFile = LizardIO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); + + /* Free resources */ + LizardIO_freeCResources(ress); + + /* Final Status */ + { clock_t const end = clock(); + double const seconds = (double)(end - start) / CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); + } + + return issueWithSrcFile; +} + + +#define FNSPACE 30 +int LizardIO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) +{ + int i; + int missed_files = 0; + char* dstFileName = (char*)malloc(FNSPACE); + size_t ofnSize = FNSPACE; + const size_t suffixSize = strlen(suffix); + cRess_t const ress = LizardIO_createCResources(); + + if (dstFileName == NULL) return ifntSize; /* not enough memory */ + + /* loop on each file */ + for (i=0; i 1 GB) { + int const seekResult = UTIL_fseek(file, 1 GB, SEEK_CUR); + if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); + storedSkips -= 1 GB; + } + + while (ptrT < bufferTEnd) { + size_t seg0SizeT = segmentSizeT; + size_t nb0T; + + /* count leading zeros */ + if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; + bufferSizeT -= seg0SizeT; + for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; + storedSkips += (unsigned)(nb0T * sizeT); + + if (nb0T != seg0SizeT) { /* not all 0s */ + int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); + storedSkips = 0; + seg0SizeT -= nb0T; + ptrT += nb0T; + { size_t const sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file); + if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block"); + } } + ptrT += seg0SizeT; + } + + if (bufferSize & maskT) { /* size not multiple of sizeT : implies end of block */ + const char* const restStart = (const char*)bufferTEnd; + const char* restPtr = restStart; + size_t const restSize = bufferSize & maskT; + const char* const restEnd = restStart + restSize; + for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; + storedSkips += (unsigned) (restPtr - restStart); + if (restPtr != restEnd) { + int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); + storedSkips = 0; + { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); + if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block"); + } } + } + + return storedSkips; +} + +static void LizardIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) +{ + if (storedSkips>0) { /* implies g_sparseFileSupport>0 */ + int const seekResult = UTIL_fseek(file, storedSkips-1, SEEK_CUR); + if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); + { const char lastZeroByte[1] = { 0 }; + size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); + if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); + } } +} + + + +typedef struct { + void* srcBuffer; + size_t srcBufferSize; + void* dstBuffer; + size_t dstBufferSize; + FILE* dstFile; + LizardF_decompressionContext_t dCtx; +} dRess_t; + +static const size_t LizardIO_dBufferSize = 64 KB; +static unsigned g_magicRead = 0; +static dRess_t LizardIO_createDResources(void) +{ + dRess_t ress; + + /* init */ + LizardF_errorCode_t const errorCode = LizardF_createDecompressionContext(&ress.dCtx, LIZARDF_VERSION); + if (LizardF_isError(errorCode)) EXM_THROW(60, "Can't create LizardF context : %s", LizardF_getErrorName(errorCode)); + + /* Allocate Memory */ + ress.srcBufferSize = LizardIO_dBufferSize; + ress.srcBuffer = malloc(ress.srcBufferSize); + ress.dstBufferSize = LizardIO_dBufferSize; + ress.dstBuffer = malloc(ress.dstBufferSize); + if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); + + ress.dstFile = NULL; + return ress; +} + +static void LizardIO_freeDResources(dRess_t ress) +{ + LizardF_errorCode_t errorCode = LizardF_freeDecompressionContext(ress.dCtx); + if (LizardF_isError(errorCode)) EXM_THROW(69, "Error : can't free LizardF context resource : %s", LizardF_getErrorName(errorCode)); + free(ress.srcBuffer); + free(ress.dstBuffer); +} + + +static unsigned long long LizardIO_decompressLizardF(dRess_t ress, FILE* srcFile, FILE* dstFile) +{ + unsigned long long filesize = 0; + LizardF_errorCode_t nextToLoad; + unsigned storedSkips = 0; + + /* Init feed with magic number (already consumed from FILE* sFile) */ + { size_t inSize = MAGICNUMBER_SIZE; + size_t outSize= 0; + LizardIO_writeLE32(ress.srcBuffer, LIZARDIO_MAGICNUMBER); + nextToLoad = LizardF_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL); + if (LizardF_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LizardF_getErrorName(nextToLoad)); + } + + /* Main Loop */ + for (;nextToLoad;) { + size_t readSize; + size_t pos = 0; + size_t decodedBytes = ress.dstBufferSize; + + /* Read input */ + if (nextToLoad > ress.srcBufferSize) nextToLoad = ress.srcBufferSize; + readSize = fread(ress.srcBuffer, 1, nextToLoad, srcFile); + if (!readSize) break; /* reached end of file or stream */ + + while ((pos < readSize) || (decodedBytes == ress.dstBufferSize)) { /* still to read, or still to flush */ + /* Decode Input (at least partially) */ + size_t remaining = readSize - pos; + decodedBytes = ress.dstBufferSize; + nextToLoad = LizardF_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); + if (LizardF_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LizardF_getErrorName(nextToLoad)); + pos += remaining; + + /* Write Block */ + if (decodedBytes) { + if (!g_testMode) + storedSkips = LizardIO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips); + filesize += decodedBytes; + DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); + } + + if (!nextToLoad) break; + } + } + /* can be out because readSize == 0, which could be an fread() error */ + if (ferror(srcFile)) EXM_THROW(67, "Read error"); + + if (!g_testMode) LizardIO_fwriteSparseEnd(dstFile, storedSkips); + if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream"); + + return filesize; +} + + +#define PTSIZE (64 KB) +#define PTSIZET (PTSIZE / sizeof(size_t)) +static unsigned long long LizardIO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) +{ + size_t buffer[PTSIZET]; + size_t readBytes = 1; + unsigned long long total = MAGICNUMBER_SIZE; + unsigned storedSkips = 0; + + size_t const sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput); + if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error"); + + while (readBytes) { + readBytes = fread(buffer, 1, PTSIZE, finput); + total += readBytes; + storedSkips = LizardIO_fwriteSparse(foutput, buffer, readBytes, storedSkips); + } + if (ferror(finput)) EXM_THROW(51, "Read Error") + + LizardIO_fwriteSparseEnd(foutput, storedSkips); + return total; +} + + +/** Safely handle cases when (unsigned)offset > LONG_MAX */ +static int fseek_u32(FILE *fp, unsigned offset, int where) +{ + const unsigned stepMax = 1U << 30; + int errorNb = 0; + + if (where != SEEK_CUR) return -1; /* Only allows SEEK_CUR */ + while (offset > 0) { + unsigned s = offset; + if (s > stepMax) s = stepMax; + errorNb = UTIL_fseek(fp, (long) s, SEEK_CUR); + if (errorNb != 0) break; + offset -= s; + } + return errorNb; +} + +#define ENDOFSTREAM ((unsigned long long)-1) +static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput) +{ + unsigned char MNstore[MAGICNUMBER_SIZE]; + unsigned magicNumber; + static unsigned nbCalls = 0; + + /* init */ + nbCalls++; + + /* Check Archive Header */ + if (g_magicRead) { /* magic number already read from finput (see legacy frame)*/ + magicNumber = g_magicRead; + g_magicRead = 0; + } else { + size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput); + if (nbReadBytes==0) { nbCalls = 0; return ENDOFSTREAM; } /* EOF */ + if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); + magicNumber = LizardIO_readLE32(MNstore); /* Little Endian format */ + } + if (LizardIO_isSkippableMagicNumber(magicNumber)) magicNumber = LIZARDIO_SKIPPABLE0; /* fold skippable magic numbers */ + + switch(magicNumber) + { + case LIZARDIO_MAGICNUMBER: + return LizardIO_decompressLizardF(ress, finput, foutput); + case LIZARDIO_SKIPPABLE0: + DISPLAYLEVEL(4, "Skipping detected skippable area \n"); + { size_t const nbReadBytes = fread(MNstore, 1, 4, finput); + if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); } + { unsigned const size = LizardIO_readLE32(MNstore); /* Little Endian format */ + int const errorNb = fseek_u32(finput, size, SEEK_CUR); + if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); } + return 0; + EXTENDED_FORMAT; /* macro extension for custom formats */ + default: + if (nbCalls == 1) { /* just started */ + if (!g_testMode && g_overwrite) { + nbCalls = 0; + return LizardIO_passThrough(finput, foutput, MNstore); + } + EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */ + } + DISPLAYLEVEL(2, "Stream followed by undecodable data\n"); + return ENDOFSTREAM; + } +} + + +static int LizardIO_decompressSrcFile(dRess_t ress, const char* input_filename, const char* output_filename) +{ + FILE* const foutput = ress.dstFile; + unsigned long long filesize = 0, decodedSize=0; + FILE* finput; + + /* Init */ + finput = LizardIO_openSrcFile(input_filename); + if (finput==NULL) return 1; + + /* Loop over multiple streams */ + do { + decodedSize = selectDecoder(ress, finput, foutput); + if (decodedSize != ENDOFSTREAM) + filesize += decodedSize; + } while (decodedSize != ENDOFSTREAM); + + /* Close */ + fclose(finput); + + if (g_removeSrcFile) { if (remove(input_filename)) EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); } /* remove source file : --rm */ + + /* Final Status */ + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "%-20.20s : decoded %llu bytes \n", input_filename, filesize); + (void)output_filename; + + return 0; +} + + +static int LizardIO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename) +{ + FILE* foutput; + + /* Init */ + foutput = LizardIO_openDstFile(output_filename); + if (foutput==NULL) return 1; /* failure */ + + ress.dstFile = foutput; + LizardIO_decompressSrcFile(ress, input_filename, output_filename); + + fclose(foutput); + + /* Copy owner, file permissions and modification time */ + { stat_t statbuf; + if (strcmp (input_filename, stdinmark) && strcmp (output_filename, stdoutmark) && UTIL_getFileStat(input_filename, &statbuf)) + UTIL_setFileStat(output_filename, &statbuf); + } + + return 0; +} + + +int LizardIO_decompressFilename(const char* input_filename, const char* output_filename) +{ + dRess_t const ress = LizardIO_createDResources(); + clock_t const start = clock(); + + int const missingFiles = LizardIO_decompressDstFile(ress, input_filename, output_filename); + + { clock_t const end = clock(); + double const seconds = (double)(end - start) / CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds); + } + + LizardIO_freeDResources(ress); + return missingFiles; +} + + +int LizardIO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix) +{ + int i; + int skippedFiles = 0; + int missingFiles = 0; + char* outFileName = (char*)malloc(FNSPACE); + size_t ofnSize = FNSPACE; + size_t const suffixSize = strlen(suffix); + dRess_t ress = LizardIO_createDResources(); + + if (outFileName==NULL) return ifntSize; /* not enough memory */ + ress.dstFile = LizardIO_openDstFile(stdoutmark); + + for (i=0; i /* size_t */ + + +/* ************************************************** */ +/* Special input/output values */ +/* ************************************************** */ +#define NULL_OUTPUT "null" +static const char stdinmark[] = "stdin"; +static const char stdoutmark[] = "stdout"; +#ifdef _WIN32 +static const char nulmark[] = "nul"; +#else +static const char nulmark[] = "/dev/null"; +#endif + + +/* ************************************************** */ +/* ****************** Functions ********************* */ +/* ************************************************** */ + +int LizardIO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel); +int LizardIO_decompressFilename(const char* input_filename, const char* output_filename); + +int LizardIO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel); +int LizardIO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix); + + +/* ************************************************** */ +/* ****************** Parameters ******************** */ +/* ************************************************** */ + +/* Default setting : overwrite = 1; + return : overwrite mode (0/1) */ +int LizardIO_setOverwrite(int yes); + +/* Default setting : testMode = 0; + return : testMode (0/1) */ +int LizardIO_setTestMode(int yes); + +/* blockSizeID : valid values : 4-5-6-7 + return : 0 if error, blockSize if OK */ +size_t LizardIO_setBlockSizeID(unsigned blockSizeID); + +/* Default setting : independent blocks */ +typedef enum { LizardIO_blockLinked=0, LizardIO_blockIndependent} LizardIO_blockMode_t; +int LizardIO_setBlockMode(LizardIO_blockMode_t blockMode); + +/* Default setting : no block checksum */ +int LizardIO_setBlockChecksumMode(int xxhash); + +/* Default setting : stream checksum enabled */ +int LizardIO_setStreamChecksumMode(int xxhash); + +/* Default setting : 0 (no notification) */ +int LizardIO_setNotificationLevel(int level); + +/* Default setting : 0 (disabled) */ +int LizardIO_setSparseFile(int enable); + +/* Default setting : 0 (disabled) */ +int LizardIO_setContentSize(int enable); + +void LizardIO_setRemoveSrcFile(unsigned flag); + + +#endif /* LIZARDIO_H_237902873 */ diff --git a/contrib/lizard/programs/platform.h b/contrib/lizard/programs/platform.h new file mode 100644 index 00000000000..51ce1ac2d14 --- /dev/null +++ b/contrib/lizard/programs/platform.h @@ -0,0 +1,145 @@ +/* + platform.h - compiler and OS detection + Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#ifndef PLATFORM_H_MODULE +#define PLATFORM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + + +/* ************************************** +* Compiler Options +****************************************/ +#if defined(_MSC_VER) +# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before and */ +# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ +# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ +# endif +#endif + + +/* ************************************** +* Detect 64-bit OS +* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros +****************************************/ +#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ + || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ + || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ + || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ + || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ + || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ + || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ +# if !defined(__64BIT__) +# define __64BIT__ 1 +# endif +#endif + + +/* ********************************************************* +* Turn on Large Files support (>4GB) for 32-bit Linux/Unix +***********************************************************/ +#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */ +# if !defined(_FILE_OFFSET_BITS) +# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ +# endif +# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ +# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ +# endif +# if defined(_AIX) || defined(__hpux) +# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ +# endif +#endif + + +/* ************************************************************ +* Detect POSIX version +* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows +* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX +* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION +***************************************************************/ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ + || defined(__midipix__) || defined(__VMS)) +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \ + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */ +# define PLATFORM_POSIX_VERSION 200112L +# else +# if defined(__linux__) || defined(__linux) +# ifndef _POSIX_C_SOURCE +# define _POSIX_C_SOURCE 200112L /* use feature test macro */ +# endif +# endif +# include /* declares _POSIX_VERSION */ +# if defined(_POSIX_VERSION) /* POSIX compliant */ +# define PLATFORM_POSIX_VERSION _POSIX_VERSION +# else +# define PLATFORM_POSIX_VERSION 0 +# endif +# endif +#endif +#if !defined(PLATFORM_POSIX_VERSION) +# define PLATFORM_POSIX_VERSION -1 +#endif + + +/*-********************************************* +* Detect if isatty() and fileno() are available +************************************************/ +#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) +# include /* isatty */ +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include /* _isatty */ +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# define IS_CONSOLE(stdStream) 0 +#endif + + +/****************************** +* OS-specific Includes +******************************/ +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) +# include /* _O_BINARY */ +# include /* _setmode, _fileno, _get_osfhandle */ +# if !defined(__DJGPP__) +# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# include /* FSCTL_SET_SPARSE */ +# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } +# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } +# else +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +# define SET_SPARSE_FILE_MODE(file) +# endif +#else +# define SET_BINARY_MODE(file) +# define SET_SPARSE_FILE_MODE(file) +#endif + + + +#if defined (__cplusplus) +} +#endif + +#endif /* PLATFORM_H_MODULE */ diff --git a/contrib/lizard/programs/util.h b/contrib/lizard/programs/util.h new file mode 100644 index 00000000000..4c2313eef00 --- /dev/null +++ b/contrib/lizard/programs/util.h @@ -0,0 +1,497 @@ +/* + util.h - utility functions + Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#ifndef UTIL_H_MODULE +#define UTIL_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + + +/*-**************************************** +* Dependencies +******************************************/ +#include "platform.h" /* PLATFORM_POSIX_VERSION */ +#include /* malloc */ +#include /* size_t, ptrdiff_t */ +#include /* fprintf */ +#include /* strlen, strncpy */ +#include /* stat, utime */ +#include /* stat */ +#if defined(_MSC_VER) +# include /* utime */ +# include /* _chmod */ +#else +# include /* chown, stat */ +# include /* utime */ +#endif +#include /* time */ +#include + + + +#ifndef UTIL_WITHOUT_BASIC_TYPES +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; +#endif +#endif + + +/* ************************************************************ +* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW +***************************************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +# define UTIL_fseek _fseeki64 +#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define UTIL_fseek fseeko +#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) +# define UTIL_fseek fseeko64 +#else +# define UTIL_fseek fseek +#endif + + +/*-**************************************** +* Sleep functions: Windows - Posix - others +******************************************/ +#if defined(_WIN32) +# include +# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) +# define UTIL_sleep(s) Sleep(1000*s) +# define UTIL_sleepMilli(milli) Sleep(milli) +#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ +# include +# include /* setpriority */ +# include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ +# if defined(PRIO_PROCESS) +# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20) +# else +# define SET_REALTIME_PRIORITY /* disabled */ +# endif +# define UTIL_sleep(s) sleep(s) +# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ +# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } +# else +# define UTIL_sleepMilli(milli) /* disabled */ +# endif +#else +# define SET_REALTIME_PRIORITY /* disabled */ +# define UTIL_sleep(s) /* disabled */ +# define UTIL_sleepMilli(milli) /* disabled */ +#endif + + +/* ************************************* +* Constants +***************************************/ +#define LIST_SIZE_INCREASE (8*1024) + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(__INTEL_COMPILER) +# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */ +#endif +#if defined(__GNUC__) +# define UTIL_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define UTIL_STATIC static inline +#elif defined(_MSC_VER) +# define UTIL_STATIC static __inline +#else +# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Time functions +******************************************/ +#if (PLATFORM_POSIX_VERSION >= 1) +#include +#include /* times */ + typedef U64 UTIL_time_t; + UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=sysconf(_SC_CLK_TCK); } + UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { struct tms junk; clock_t newTicks = (clock_t) times(&junk); (void)junk; *x = (UTIL_time_t)newTicks; } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / ticksPerSecond; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / ticksPerSecond; } +#elif defined(_WIN32) /* Windows */ + typedef LARGE_INTEGER UTIL_time_t; + UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); } + UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } +#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */ + typedef clock_t UTIL_time_t; + UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; } + UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } +#endif + + +/* returns time span in microseconds */ +UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond ) +{ + UTIL_time_t clockEnd; + UTIL_getTime(&clockEnd); + return UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd); +} + + +UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) +{ + UTIL_time_t clockStart, clockEnd; + UTIL_getTime(&clockStart); + do { + UTIL_getTime(&clockEnd); + } while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); +} + + + +/*-**************************************** +* File functions +******************************************/ +#if defined(_MSC_VER) + #define chmod _chmod + typedef struct __stat64 stat_t; +#else + typedef struct stat stat_t; +#endif + + +UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) +{ + int res = 0; + struct utimbuf timebuf; + + timebuf.actime = time(NULL); + timebuf.modtime = statbuf->st_mtime; + res += utime(filename, &timebuf); /* set access and modification times */ + +#if !defined(_WIN32) + res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */ +#endif + + res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */ + + errno = 0; + return -res; /* number of errors is returned */ +} + + +UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) +{ + int r; +#if defined(_MSC_VER) + r = _stat64(infilename, statbuf); + if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ +#else + r = stat(infilename, statbuf); + if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */ +#endif + return 1; +} + + +UTIL_STATIC int UTIL_isRegFile(const char* infilename) +{ + stat_t statbuf; + return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */ +} + + +UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) +{ + int r; + stat_t statbuf; +#if defined(_MSC_VER) + r = _stat64(infilename, &statbuf); + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; +#else + r = stat(infilename, &statbuf); + if (!r && S_ISDIR(statbuf.st_mode)) return 1; +#endif + return 0; +} + + +UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct __stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#elif defined(__MINGW32__) && defined (__MSVCRT__) + struct _stati64 statbuf; + r = _stati64(infilename, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ +#endif + return (U64)statbuf.st_size; +} + + +UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) +{ + U64 total = 0; + unsigned n; + for (n=0; n= *bufEnd) { + ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; + *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); + *bufEnd = *bufStart + newListSize; + if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } + } + if (*bufStart + *pos + pathLength < *bufEnd) { + strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos)); + *pos += pathLength + 1; + nbFiles++; + } + } + free(path); + } while (FindNextFileA(hFile, &cFile)); + + FindClose(hFile); + return nbFiles; +} + +#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ +# define UTIL_HAS_CREATEFILELIST +# include /* opendir, readdir */ +# include /* strerror, memcpy */ + +UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +{ + DIR *dir; + struct dirent *entry; + char* path; + int dirLength, fnameLength, pathLength, nbFiles = 0; + + if (!(dir = opendir(dirName))) { + fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); + return 0; + } + + dirLength = (int)strlen(dirName); + errno = 0; + while ((entry = readdir(dir)) != NULL) { + if (strcmp (entry->d_name, "..") == 0 || + strcmp (entry->d_name, ".") == 0) continue; + fnameLength = (int)strlen(entry->d_name); + path = (char*) malloc(dirLength + fnameLength + 2); + if (!path) { closedir(dir); return 0; } + memcpy(path, dirName, dirLength); + path[dirLength] = '/'; + memcpy(path+dirLength+1, entry->d_name, fnameLength); + pathLength = dirLength+1+fnameLength; + path[pathLength] = 0; + + if (UTIL_isDirectory(path)) { + nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */ + if (*bufStart == NULL) { free(path); closedir(dir); return 0; } + } else { + if (*bufStart + *pos + pathLength >= *bufEnd) { + ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; + *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); + *bufEnd = *bufStart + newListSize; + if (*bufStart == NULL) { free(path); closedir(dir); return 0; } + } + if (*bufStart + *pos + pathLength < *bufEnd) { + strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos)); + *pos += pathLength + 1; + nbFiles++; + } + } + free(path); + errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */ + } + + if (errno != 0) { + fprintf(stderr, "readdir(%s) error: %s\n", dirName, strerror(errno)); + free(*bufStart); + *bufStart = NULL; + } + closedir(dir); + return nbFiles; +} + +#else + +UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) +{ + (void)bufStart; (void)bufEnd; (void)pos; + fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); + return 0; +} + +#endif /* #ifdef _WIN32 */ + +/* + * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories, + * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb). + * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) + * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. + */ +UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb) +{ + size_t pos; + unsigned i, nbFiles; + char* buf = (char*)malloc(LIST_SIZE_INCREASE); + char* bufend = buf + LIST_SIZE_INCREASE; + const char** fileTable; + + if (!buf) return NULL; + + for (i=0, pos=0, nbFiles=0; i= bufend) { + ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; + buf = (char*)UTIL_realloc(buf, newListSize); + bufend = buf + newListSize; + if (!buf) return NULL; + } + if (buf + pos + len < bufend) { + strncpy(buf + pos, inputNames[i], bufend - (buf + pos)); + pos += len + 1; + nbFiles++; + } + } else { + nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend); + if (buf == NULL) return NULL; + } } + + if (nbFiles == 0) { free(buf); return NULL; } + + fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); + if (!fileTable) { free(buf); return NULL; } + + for (i=0, pos=0; i bufend) { free(buf); free((void*)fileTable); return NULL; } + + *allocatedBuffer = buf; + *allocatedNamesNb = nbFiles; + + return fileTable; +} + + +UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer) +{ + if (allocatedBuffer) free(allocatedBuffer); + if (filenameTable) free((void*)filenameTable); +} + + +#if defined (__cplusplus) +} +#endif + +#endif /* UTIL_H_MODULE */ diff --git a/contrib/lizard/tests/.gitignore b/contrib/lizard/tests/.gitignore new file mode 100644 index 00000000000..06891acf794 --- /dev/null +++ b/contrib/lizard/tests/.gitignore @@ -0,0 +1,13 @@ + +# test build artefacts +datagen +frametest +frametest32 +fullbench +fullbench32 +fuzzer +fuzzer32 +fasttest + +# test artefacts +tmp* diff --git a/contrib/lizard/tests/COPYING b/contrib/lizard/tests/COPYING new file mode 100644 index 00000000000..d159169d105 --- /dev/null +++ b/contrib/lizard/tests/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/contrib/lizard/tests/README.md b/contrib/lizard/tests/README.md new file mode 100644 index 00000000000..4dc2edc4396 --- /dev/null +++ b/contrib/lizard/tests/README.md @@ -0,0 +1,15 @@ +Programs and scripts for automated testing of Lizard +======================================================= + +This directory contains the following programs and scripts: +- `datagen` : Synthetic and parametrable data generator, for tests +- `frametest` : Test tool that checks lizard_frame integrity on target platform +- `fullbench` : Precisely measure speed for each lizard inner functions +- `fuzzer` : Test tool, to check lizard integrity on target platform + + +#### License + +All files in this directory are licensed under GPL-v2. +See [COPYING](COPYING) for details. +The text of the license is also included at the top of each source file. diff --git a/contrib/lizard/tests/datagencli.c b/contrib/lizard/tests/datagencli.c new file mode 100644 index 00000000000..47c79d31522 --- /dev/null +++ b/contrib/lizard/tests/datagencli.c @@ -0,0 +1,192 @@ +/* + datagencli.c + compressible data command line generator + Copyright (C) Yann Collet 2012-2015 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + +/************************************** +* Includes +**************************************/ +#include /* fprintf, stderr */ +#include "datagen.h" /* RDG_generate */ + + +/************************************** +* Basic Types +**************************************/ +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +/************************************** +* Constants +**************************************/ +#ifndef ZSTD_VERSION +# define ZSTD_VERSION "r1" +#endif + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define SIZE_DEFAULT (64 KB) +#define SEED_DEFAULT 0 +#define COMPRESSIBILITY_DEFAULT 50 + + +/************************************** +* Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static unsigned displayLevel = 2; + + +/********************************************************* +* Command line +*********************************************************/ +static int usage(char* programName) +{ + DISPLAY( "Compressible data generator\n"); + DISPLAY( "Usage :\n"); + DISPLAY( " %s [size] [args]\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); + DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT); + DISPLAY( " -h : display help and exit\n"); + DISPLAY( "Special values :\n"); + DISPLAY( " -P0 : generate incompressible noise\n"); + DISPLAY( " -P100 : generate sparse files\n"); + return 0; +} + + +int main(int argc, char** argv) +{ + int argNb; + double proba = (double)COMPRESSIBILITY_DEFAULT / 100; + double litProba = 0.0; + U64 size = SIZE_DEFAULT; + U32 seed = SEED_DEFAULT; + char* programName; + + /* Check command line */ + programName = argv[0]; + for(argNb=1; argNb='0') && (*argument<='9')) + { + size *= 10; + size += *argument - '0'; + argument++; + } + if (*argument=='K') { size <<= 10; argument++; } + if (*argument=='M') { size <<= 20; argument++; } + if (*argument=='G') { size <<= 30; argument++; } + if (*argument=='B') { argument++; } + break; + case 's': + argument++; + seed=0; + while ((*argument>='0') && (*argument<='9')) + { + seed *= 10; + seed += *argument - '0'; + argument++; + } + break; + case 'P': + argument++; + proba=0.0; + while ((*argument>='0') && (*argument<='9')) + { + proba *= 10; + proba += *argument - '0'; + argument++; + } + if (proba>100.) proba=100.; + proba /= 100.; + break; + case 'L': /* hidden argument : Literal distribution probability */ + argument++; + litProba=0.; + while ((*argument>='0') && (*argument<='9')) + { + litProba *= 10; + litProba += *argument - '0'; + argument++; + } + if (litProba>100.) litProba=100.; + litProba /= 100.; + break; + case 'v': + displayLevel = 4; + argument++; + break; + default: + return usage(programName); + } + } + + } + } + + DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION); + DISPLAYLEVEL(3, "Seed = %u \n", seed); + if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100)); + + RDG_genOut(size, proba, litProba, seed); + DISPLAYLEVEL(1, "\n"); + + return 0; +} diff --git a/contrib/lizard/tests/frametest.c b/contrib/lizard/tests/frametest.c new file mode 100644 index 00000000000..8fe64869fa4 --- /dev/null +++ b/contrib/lizard/tests/frametest.c @@ -0,0 +1,866 @@ +/* + frameTest - test tool for lizard_frame + Copyright (C) Yann Collet 2014-2016 + Copyright (C) Przemyslaw Skibinski 2016-2017 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + +/*-************************************ +* Compiler specific +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +#endif + + +/*-************************************ +* Includes +**************************************/ +#include /* malloc, free */ +#include /* fprintf */ +#include /* strcmp */ +#include /* clock_t, clock(), CLOCKS_PER_SEC */ +#include "lizard_frame_static.h" +#include "lizard_compress.h" /* LIZARD_VERSION_STRING */ +#define XXH_STATIC_LINKING_ONLY +#include "xxhash/xxhash.h" /* XXH64 */ + + +/*-************************************ +* Basic Types +**************************************/ +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +#else +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; +#endif + + +/* unoptimized version; solves endianess & alignment issues */ +static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) +{ + BYTE* dstPtr = (BYTE*)dstVoidPtr; + dstPtr[0] = (BYTE)value32; + dstPtr[1] = (BYTE)(value32 >> 8); + dstPtr[2] = (BYTE)(value32 >> 16); + dstPtr[3] = (BYTE)(value32 >> 24); +} + + +/*-************************************ +* Constants +**************************************/ +#define LIZARDF_MAGIC_SKIPPABLE_START 0x184D2A50U + +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +static const U32 nbTestsDefault = 256 KB; +#define COMPRESSIBLE_NOISE_LENGTH (2 MB) +#define FUZ_COMPRESSIBILITY_DEFAULT 50 +static const U32 prime1 = 2654435761U; +static const U32 prime2 = 2246822519U; + + + +/*-************************************ +* Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \ + if ((FUZ_GetClockSpan(g_clockTime) > refreshRate) || (displayLevel>=4)) \ + { g_clockTime = clock(); DISPLAY(__VA_ARGS__); \ + if (displayLevel>=4) fflush(stdout); } } +static const clock_t refreshRate = CLOCKS_PER_SEC / 6; +static clock_t g_clockTime = 0; + + +/*-*************************************** +* Local Parameters +*****************************************/ +static U32 no_prompt = 0; +static char* programName; +static U32 displayLevel = 2; +static U32 pause = 0; + + +/*-******************************************************* +* Fuzzer functions +*********************************************************/ +static clock_t FUZ_GetClockSpan(clock_t clockStart) +{ + return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */ +} + + +#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +unsigned int FUZ_rand(unsigned int* src) +{ + U32 rand32 = *src; + rand32 *= prime1; + rand32 += prime2; + rand32 = FUZ_rotl32(rand32, 13); + *src = rand32; + return rand32 >> 5; +} + + +#define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF) +#define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) +static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, double proba, U32* seed) +{ + BYTE* BBuffer = (BYTE*)buffer; + unsigned pos = 0; + U32 P32 = (U32)(32768 * proba); + + /* First Byte */ + BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); + + while (pos < bufferSize) { + /* Select : Literal (noise) or copy (within 64K) */ + if (FUZ_RAND15BITS < P32) { + /* Copy (within 64K) */ + unsigned match, end; + unsigned length = FUZ_RANDLENGTH + 4; + unsigned offset = FUZ_RAND15BITS + 1; + if (offset > pos) offset = pos; + if (pos + length > bufferSize) length = bufferSize - pos; + match = pos - offset; + end = pos + length; + while (pos < end) BBuffer[pos++] = BBuffer[match++]; + } else { + /* Literal (noise) */ + unsigned end; + unsigned length = FUZ_RANDLENGTH; + if (pos + length > bufferSize) length = bufferSize - pos; + end = pos + length; + while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); + } + } +} + + +static unsigned FUZ_highbit(U32 v32) +{ + unsigned nbBits = 0; + if (v32==0) return 0; + while (v32) v32 >>= 1, nbBits ++; + return nbBits; +} + + +int basicTests(U32 seed, double compressibility) +{ + int testResult = 0; + void* CNBuffer; + void* compressedBuffer; + void* decodedBuffer; + U32 randState = seed; + size_t cSize, testSize; + LizardF_preferences_t prefs; + LizardF_decompressionContext_t dCtx = NULL; + LizardF_compressionContext_t cctx = NULL; + U64 crcOrig; + + /* Create compressible test buffer */ + memset(&prefs, 0, sizeof(prefs)); + CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + compressedBuffer = malloc(LizardF_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL)); + decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); + crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + + /* Special case : null-content frame */ + testSize = 0; + DISPLAYLEVEL(3, "LizardF_compressFrame, compress null content : \n"); + cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); + if (LizardF_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize); + + DISPLAYLEVEL(3, "LizardF_createDecompressionContext \n"); + { LizardF_errorCode_t const errorCode = LizardF_createDecompressionContext(&dCtx, LIZARDF_VERSION); + if (LizardF_isError(errorCode)) goto _output_error; } + + DISPLAYLEVEL(3, "LizardF_getFrameInfo on null-content frame (#157) \n"); + { size_t avail_in = cSize; + LizardF_frameInfo_t frame_info; + LizardF_errorCode_t const errorCode = LizardF_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in); + if (LizardF_isError(errorCode)) goto _output_error; + } + + DISPLAYLEVEL(3, "LizardF_freeDecompressionContext \n"); + { LizardF_errorCode_t const errorCode = LizardF_freeDecompressionContext(dCtx); + if (LizardF_isError(errorCode)) goto _output_error; } + + /* Trivial tests : one-step frame */ + testSize = COMPRESSIBLE_NOISE_LENGTH; + DISPLAYLEVEL(3, "LizardF_compressFrame, using default preferences : \n"); + cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); + if (LizardF_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + + DISPLAYLEVEL(3, "Decompression test : \n"); + { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; + size_t compressedBufferSize = cSize; + BYTE* op = (BYTE*)decodedBuffer; + BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; + BYTE* ip = (BYTE*)compressedBuffer; + BYTE* const iend = (BYTE*)compressedBuffer + cSize; + U64 crcDest; + + LizardF_errorCode_t errorCode = LizardF_createDecompressionContext(&dCtx, LIZARDF_VERSION); + if (LizardF_isError(errorCode)) goto _output_error; + + DISPLAYLEVEL(3, "Single Block : \n"); + errorCode = LizardF_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL); + if (LizardF_isError(errorCode)) goto _output_error; + crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; + DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); + + DISPLAYLEVEL(3, "Reusing decompression context \n"); + { size_t iSize = compressedBufferSize - 4; + const BYTE* cBuff = (const BYTE*) compressedBuffer; + size_t decResult; + DISPLAYLEVEL(3, "Missing last 4 bytes : "); + decResult = LizardF_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL); + if (LizardF_isError(decResult)) goto _output_error; + if (!decResult) goto _output_error; /* not finished */ + DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned)decResult); + cBuff += iSize; + iSize = decResult; + decResult = LizardF_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL); + if (decResult != 0) goto _output_error; /* should finish now */ + crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; + } + + { size_t oSize = 0; + size_t iSize = 0; + LizardF_frameInfo_t fi; + + DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); + errorCode = LizardF_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL); + if (LizardF_isError(errorCode)) goto _output_error; + DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); + + DISPLAYLEVEL(3, "get FrameInfo on null input : "); + errorCode = LizardF_getFrameInfo(dCtx, &fi, ip, &iSize); + if (errorCode != (size_t)-LizardF_ERROR_frameHeader_incomplete) goto _output_error; + DISPLAYLEVEL(3, " correctly failed : %s \n", LizardF_getErrorName(errorCode)); + + DISPLAYLEVEL(3, "get FrameInfo on not enough input : "); + iSize = 6; + errorCode = LizardF_getFrameInfo(dCtx, &fi, ip, &iSize); + if (errorCode != (size_t)-LizardF_ERROR_frameHeader_incomplete) goto _output_error; + DISPLAYLEVEL(3, " correctly failed : %s \n", LizardF_getErrorName(errorCode)); + ip += iSize; + + DISPLAYLEVEL(3, "get FrameInfo on enough input : "); + iSize = 15 - iSize; + errorCode = LizardF_getFrameInfo(dCtx, &fi, ip, &iSize); + if (LizardF_isError(errorCode)) goto _output_error; + DISPLAYLEVEL(3, " correctly decoded \n"); + ip += iSize; + } + + DISPLAYLEVEL(3, "Byte after byte : \n"); + while (ip < iend) { + size_t oSize = oend-op; + size_t iSize = 1; + errorCode = LizardF_decompress(dCtx, op, &oSize, ip, &iSize, NULL); + if (LizardF_isError(errorCode)) goto _output_error; + op += oSize; + ip += iSize; + } + crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; + DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-(BYTE*)decodedBuffer), COMPRESSIBLE_NOISE_LENGTH); + + errorCode = LizardF_freeDecompressionContext(dCtx); + if (LizardF_isError(errorCode)) goto _output_error; + } + + DISPLAYLEVEL(3, "Using 128 KB block : \n"); + prefs.frameInfo.blockSizeID = LizardF_max128KB; + prefs.frameInfo.contentChecksumFlag = LizardF_contentChecksumEnabled; + cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); + if (LizardF_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + + DISPLAYLEVEL(3, "without checksum : \n"); + prefs.frameInfo.contentChecksumFlag = LizardF_noContentChecksum; + cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); + if (LizardF_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + + DISPLAYLEVEL(3, "Using 256 KB block : \n"); + prefs.frameInfo.blockSizeID = LizardF_max256KB; + prefs.frameInfo.contentChecksumFlag = LizardF_contentChecksumEnabled; + cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); + if (LizardF_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + + DISPLAYLEVEL(3, "Decompression test : \n"); + { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; + unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); + BYTE* op = (BYTE*)decodedBuffer; + BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; + BYTE* ip = (BYTE*)compressedBuffer; + BYTE* const iend = (BYTE*)compressedBuffer + cSize; + U64 crcDest; + + LizardF_errorCode_t errorCode = LizardF_createDecompressionContext(&dCtx, LIZARDF_VERSION); + if (LizardF_isError(errorCode)) goto _output_error; + + DISPLAYLEVEL(3, "random segment sizes : \n"); + while (ip < iend) { + unsigned nbBits = FUZ_rand(&randState) % maxBits; + size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; + //DISPLAY("%7i : + %6i\n", (int)(ip-(BYTE*)compressedBuffer), (int)iSize); + errorCode = LizardF_decompress(dCtx, op, &oSize, ip, &iSize, NULL); + if (LizardF_isError(errorCode)) goto _output_error; + op += oSize; + ip += iSize; + } + crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; + DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); + + errorCode = LizardF_freeDecompressionContext(dCtx); + if (LizardF_isError(errorCode)) goto _output_error; + } + + DISPLAYLEVEL(3, "without checksum : \n"); + prefs.frameInfo.contentChecksumFlag = LizardF_noContentChecksum; + cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); + if (LizardF_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + + DISPLAYLEVEL(3, "Using 1 MB block : \n"); + prefs.frameInfo.blockSizeID = LizardF_max1MB; + prefs.frameInfo.contentChecksumFlag = LizardF_contentChecksumEnabled; + cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); + if (LizardF_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + + DISPLAYLEVEL(3, "without checksum : \n"); + prefs.frameInfo.contentChecksumFlag = LizardF_noContentChecksum; + cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); + if (LizardF_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + + DISPLAYLEVEL(3, "Using 4 MB block : \n"); + prefs.frameInfo.blockSizeID = LizardF_max4MB; + prefs.frameInfo.contentChecksumFlag = LizardF_contentChecksumEnabled; + cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); + if (LizardF_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + + DISPLAYLEVEL(3, "without checksum : \n"); + prefs.frameInfo.contentChecksumFlag = LizardF_noContentChecksum; + cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); + if (LizardF_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); + + { size_t errorCode; + BYTE* const ostart = (BYTE*)compressedBuffer; + BYTE* op = ostart; + errorCode = LizardF_createCompressionContext(&cctx, LIZARDF_VERSION); + if (LizardF_isError(errorCode)) goto _output_error; + + DISPLAYLEVEL(3, "compress without frameSize : \n"); + memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo)); + errorCode = LizardF_compressBegin(cctx, compressedBuffer, testSize, &prefs); + if (LizardF_isError(errorCode)) goto _output_error; + op += errorCode; + errorCode = LizardF_compressUpdate(cctx, op, LizardF_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); + if (LizardF_isError(errorCode)) goto _output_error; + op += errorCode; + errorCode = LizardF_compressEnd(cctx, compressedBuffer, testSize, NULL); + if (LizardF_isError(errorCode)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); + + DISPLAYLEVEL(3, "compress with frameSize : \n"); + prefs.frameInfo.contentSize = testSize; + op = ostart; + errorCode = LizardF_compressBegin(cctx, compressedBuffer, testSize, &prefs); + if (LizardF_isError(errorCode)) goto _output_error; + op += errorCode; + errorCode = LizardF_compressUpdate(cctx, op, LizardF_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); + if (LizardF_isError(errorCode)) goto _output_error; + op += errorCode; + errorCode = LizardF_compressEnd(cctx, compressedBuffer, testSize, NULL); + if (LizardF_isError(errorCode)) goto _output_error; + DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); + + DISPLAYLEVEL(3, "compress with wrong frameSize : \n"); + prefs.frameInfo.contentSize = testSize+1; + op = ostart; + errorCode = LizardF_compressBegin(cctx, compressedBuffer, testSize, &prefs); + if (LizardF_isError(errorCode)) goto _output_error; + op += errorCode; + errorCode = LizardF_compressUpdate(cctx, op, LizardF_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); + if (LizardF_isError(errorCode)) goto _output_error; + op += errorCode; + errorCode = LizardF_compressEnd(cctx, op, testSize, NULL); + if (LizardF_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LizardF_getErrorName(errorCode)); } + else + goto _output_error; + + errorCode = LizardF_freeCompressionContext(cctx); + if (LizardF_isError(errorCode)) goto _output_error; + cctx = NULL; + } + + DISPLAYLEVEL(3, "Skippable frame test : \n"); + { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; + unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); + BYTE* op = (BYTE*)decodedBuffer; + BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; + BYTE* ip = (BYTE*)compressedBuffer; + BYTE* iend = (BYTE*)compressedBuffer + cSize + 8; + + LizardF_errorCode_t errorCode = LizardF_createDecompressionContext(&dCtx, LIZARDF_VERSION); + if (LizardF_isError(errorCode)) goto _output_error; + + /* generate skippable frame */ + FUZ_writeLE32(ip, LIZARDF_MAGIC_SKIPPABLE_START); + FUZ_writeLE32(ip+4, (U32)cSize); + + DISPLAYLEVEL(3, "random segment sizes : \n"); + while (ip < iend) { + unsigned nbBits = FUZ_rand(&randState) % maxBits; + size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; + errorCode = LizardF_decompress(dCtx, op, &oSize, ip, &iSize, NULL); + if (LizardF_isError(errorCode)) goto _output_error; + op += oSize; + ip += iSize; + } + DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize); + + /* generate zero-size skippable frame */ + DISPLAYLEVEL(3, "zero-size skippable frame\n"); + ip = (BYTE*)compressedBuffer; + op = (BYTE*)decodedBuffer; + FUZ_writeLE32(ip, LIZARDF_MAGIC_SKIPPABLE_START+1); + FUZ_writeLE32(ip+4, 0); + iend = ip+8; + + while (ip < iend) { + unsigned nbBits = FUZ_rand(&randState) % maxBits; + size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; + errorCode = LizardF_decompress(dCtx, op, &oSize, ip, &iSize, NULL); + if (LizardF_isError(errorCode)) goto _output_error; + op += oSize; + ip += iSize; + } + DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); + + DISPLAYLEVEL(3, "Skippable frame header complete in first call \n"); + ip = (BYTE*)compressedBuffer; + op = (BYTE*)decodedBuffer; + FUZ_writeLE32(ip, LIZARDF_MAGIC_SKIPPABLE_START+2); + FUZ_writeLE32(ip+4, 10); + iend = ip+18; + while (ip < iend) { + size_t iSize = 10; + size_t oSize = 10; + if (iSize > (size_t)(iend-ip)) iSize = iend-ip; + errorCode = LizardF_decompress(dCtx, op, &oSize, ip, &iSize, NULL); + if (LizardF_isError(errorCode)) goto _output_error; + op += oSize; + ip += iSize; + } + DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); + } + + DISPLAY("Basic tests completed \n"); +_end: + free(CNBuffer); + free(compressedBuffer); + free(decodedBuffer); + LizardF_freeDecompressionContext(dCtx); dCtx = NULL; + LizardF_freeCompressionContext(cctx); cctx = NULL; + return testResult; + +_output_error: + testResult = 1; + DISPLAY("Error detected ! \n"); + goto _end; +} + + +static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous) +{ + int p=0; + const BYTE* b1=(const BYTE*)buff1; + const BYTE* b2=(const BYTE*)buff2; + if (nonContiguous) { + DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size); + return; + } + while (b1[p]==b2[p]) p++; + DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]); +} + + +static const U32 srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ + +int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s) +{ + unsigned testResult = 0; + unsigned testNb = 0; + void* srcBuffer = NULL; + void* compressedBuffer = NULL; + void* decodedBuffer = NULL; + U32 coreRand = seed; + LizardF_decompressionContext_t dCtx = NULL; + LizardF_compressionContext_t cCtx = NULL; + size_t result; + clock_t const startClock = clock(); + clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; + XXH64_state_t xxh64; +# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ + DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } + + /* Create buffers */ + result = LizardF_createDecompressionContext(&dCtx, LIZARDF_VERSION); + CHECK(LizardF_isError(result), "Allocation failed (error %i)", (int)result); + result = LizardF_createCompressionContext(&cCtx, LIZARDF_VERSION); + CHECK(LizardF_isError(result), "Allocation failed (error %i)", (int)result); + srcBuffer = malloc(srcDataLength); + CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); + compressedBuffer = malloc(LizardF_compressFrameBound(srcDataLength, NULL)); + CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); + decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ + CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); + FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); + + /* jump to requested testNb */ + for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer + + /* main fuzzer test loop */ + for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) { + U32 randState = coreRand ^ prime1; + unsigned BSId = 4 + (FUZ_rand(&randState) & 3); + unsigned BMId = FUZ_rand(&randState) & 1; + unsigned CCflag = FUZ_rand(&randState) & 1; + unsigned autoflush = (FUZ_rand(&randState) & 7) == 2; + LizardF_preferences_t prefs; + LizardF_compressOptions_t cOptions; + LizardF_decompressOptions_t dOptions; + unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1; + size_t srcSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; + cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); + + result = LizardF_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); + CHECK(LizardF_isError(result), "Compression failed (error %i) iSize=%d oSize=%d", (int)result, (int)iSize, (int)oSize); + op += result; + ip += iSize; + + if (forceFlush) { + result = LizardF_flush(cCtx, op, oend-op, &cOptions); + CHECK(LizardF_isError(result), "Compression flush failed (error %i)", (int)result); + op += result; + } + } + result = LizardF_compressEnd(cCtx, op, oend-op, &cOptions); + CHECK(LizardF_isError(result), "Compression completion failed (error %i)", (int)result); + op += result; + cSize = op-(BYTE*)compressedBuffer; + } + + { const BYTE* ip = (const BYTE*)compressedBuffer; + const BYTE* const iend = ip + cSize; + BYTE* op = (BYTE*)decodedBuffer; + BYTE* const oend = op + srcDataLength; + size_t totalOut = 0; + unsigned maxBits = FUZ_highbit((U32)cSize); + unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1; + nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst; /* 0=>0; 1=>1,2 */ + XXH64_reset(&xxh64, 1); + if (maxBits < 3) maxBits = 3; + while (ip < iend) { + unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; + unsigned nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; + size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; + if (oSize > (size_t)(oend-op)) oSize = oend-op; + dOptions.stableDst = FUZ_rand(&randState) & 1; + if (nonContiguousDst==2) dOptions.stableDst = 0; + result = LizardF_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); + if (result == (size_t)-LizardF_ERROR_contentChecksum_invalid) + locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); + CHECK(LizardF_isError(result), "Decompression failed (error %i:%s)", (int)result, LizardF_getErrorName((LizardF_errorCode_t)result)); + XXH64_update(&xxh64, op, (U32)oSize); + totalOut += oSize; + op += oSize; + ip += iSize; + op += nonContiguousDst; + if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */ + } + CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); + if (totalOut) { /* otherwise, it's a skippable frame */ + crcDecoded = XXH64_digest(&xxh64); + if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); + CHECK(crcDecoded != crcOrig, "Decompression corruption"); + } + } + } + + DISPLAYLEVEL(2, "\rAll tests completed \n"); + +_end: + LizardF_freeDecompressionContext(dCtx); + LizardF_freeCompressionContext(cCtx); + free(srcBuffer); + free(compressedBuffer); + free(decodedBuffer); + + if (pause) { + DISPLAY("press enter to finish \n"); + (void)getchar(); + } + return testResult; + +_output_error: + testResult = 1; + goto _end; +} + + +int FUZ_usage(void) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args]\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); + DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); + DISPLAY( " -s# : Select seed (default:prompt user)\n"); + DISPLAY( " -t# : Select starting test number (default:0)\n"); + DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); + DISPLAY( " -v : verbose\n"); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + + +int main(int argc, char** argv) +{ + U32 seed=0; + int seedset=0; + int argNb; + int nbTests = nbTestsDefault; + int testNb = 0; + int proba = FUZ_COMPRESSIBILITY_DEFAULT; + int result=0; + U32 duration=0; + + /* Check command line */ + programName = argv[0]; + for(argNb=1; argNb='0') && (*argument<='9')) { + nbTests *= 10; + nbTests += *argument - '0'; + argument++; + } + break; + + case 'T': + argument++; + nbTests = 0; duration = 0; + for (;;) { + switch(*argument) + { + case 'm': duration *= 60; argument++; continue; + case 's': + case 'n': argument++; continue; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': duration *= 10; duration += *argument++ - '0'; continue; + } + break; + } + break; + + case 's': + argument++; + seed=0; + seedset=1; + while ((*argument>='0') && (*argument<='9')) { + seed *= 10; + seed += *argument - '0'; + argument++; + } + break; + case 't': + argument++; + testNb=0; + while ((*argument>='0') && (*argument<='9')) { + testNb *= 10; + testNb += *argument - '0'; + argument++; + } + break; + case 'P': /* compressibility % */ + argument++; + proba=0; + while ((*argument>='0') && (*argument<='9')) { + proba *= 10; + proba += *argument - '0'; + argument++; + } + if (proba<0) proba=0; + if (proba>100) proba=100; + break; + default: + ; + return FUZ_usage(); + } + } + } + } + + /* Get Seed */ + printf("Starting lizard_frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LIZARD_VERSION_STRING); + + if (!seedset) { + time_t const t = time(NULL); + U32 const h = XXH32(&t, sizeof(t), 1); + seed = h % 10000; + } + printf("Seed = %u\n", seed); + if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); + + if (nbTests<=0) nbTests=1; + + if (testNb==0) result = basicTests(seed, ((double)proba) / 100); + if (result) return 1; + return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration); +} diff --git a/contrib/lizard/tests/fullbench.c b/contrib/lizard/tests/fullbench.c new file mode 100644 index 00000000000..d9070d13345 --- /dev/null +++ b/contrib/lizard/tests/fullbench.c @@ -0,0 +1,732 @@ +/* + bench.c - Demo program to benchmark open-source compression algorithm + Copyright (C) Yann Collet 2012-2015 + Copyright (C) Przemyslaw Skibinski 2016-2017 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - Lizard source repository : https://github.com/inikep/lizard +*/ + +/************************************** +* Compiler Options +**************************************/ +/* Disable some Visual warning messages */ +#define _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ + +/* Unix Large Files support (>4GB) */ +#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions +# define _LARGEFILE_SOURCE +# define _FILE_OFFSET_BITS 64 +#elif ! defined(__LP64__) // No point defining Large file for 64 bit +# define _LARGEFILE64_SOURCE +#endif + +// S_ISREG & gettimeofday() are not supported by MSVC +#if defined(_MSC_VER) || defined(_WIN32) +# define BMK_LEGACY_TIMER 1 +#endif + + +/************************************** +* Includes +**************************************/ +#include /* malloc, free */ +#include /* fprintf, fopen, ftello64 */ +#include /* stat64 */ +#include /* stat64 */ +#include /* strcmp */ +#include /* clock_t, clock(), CLOCKS_PER_SEC */ + +#include "lizard_compress.h" +#include "lizard_decompress.h" +#include "lizard_common.h" /* Lizard_compress_MinLevel, Lizard_createStream_MinLevel */ +#include "lizard_frame.h" + +#include "xxhash/xxhash.h" + + +/************************************** +* Compiler Options +**************************************/ +/* S_ISREG & gettimeofday() are not supported by MSVC */ +#if !defined(S_ISREG) +# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) +#endif + + + +/************************************** +* Constants +**************************************/ +#define PROGRAM_DESCRIPTION "Lizard speed analyzer" +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s v%s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, LIZARD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR + +#define NBLOOPS 6 +#define TIMELOOP (CLOCKS_PER_SEC * 25 / 10) + +#define KNUTH 2654435761U +#define MAX_MEM (1920 MB) +#define DEFAULT_CHUNKSIZE (4 MB) + +#define ALL_COMPRESSORS 0 +#define ALL_DECOMPRESSORS 0 + + +/************************************** +* Local structures +**************************************/ +struct chunkParameters +{ + U32 id; + char* origBuffer; + char* compressedBuffer; + int origSize; + int compressedSize; +}; + + +/************************************** +* Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define PROGRESS(...) g_noPrompt ? 0 : DISPLAY(__VA_ARGS__) + + +/************************************** +* Benchmark Parameters +**************************************/ +static int g_chunkSize = DEFAULT_CHUNKSIZE; +static int g_nbIterations = NBLOOPS; +static int g_pause = 0; +static int g_compressionTest = 1; +static int g_compressionAlgo = ALL_COMPRESSORS; +static int g_decompressionTest = 1; +static int g_decompressionAlgo = ALL_DECOMPRESSORS; +static int g_noPrompt = 0; + +static void BMK_setBlocksize(int bsize) +{ + g_chunkSize = bsize; + DISPLAY("-Using Block Size of %i KB-\n", g_chunkSize>>10); +} + +static void BMK_setNbIterations(int nbLoops) +{ + g_nbIterations = nbLoops; + DISPLAY("- %i iterations -\n", g_nbIterations); +} + +static void BMK_setPause(void) +{ + g_pause = 1; +} + + +/********************************************************* +* Private functions +*********************************************************/ +static clock_t BMK_GetClockSpan( clock_t clockStart ) +{ + return clock() - clockStart; /* works even if overflow; max span ~30 mn */ +} + + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t step = 64 MB; + BYTE* testmem=NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += 2*step; + if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; + + while (!testmem) { + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + testmem = (BYTE*) malloc ((size_t)requiredMem); + } + free (testmem); + + /* keep some space available */ + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + + return (size_t)requiredMem; +} + + +static U64 BMK_GetFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ + return (U64)statbuf.st_size; +} + + +/********************************************************* +* Benchmark function +*********************************************************/ +Lizard_stream_t *Lizard_stream; +static void local_Lizard_createStream(void) +{ + Lizard_stream = Lizard_resetStream_MinLevel(Lizard_stream); +} + +static int local_Lizard_saveDict(const char* in, char* out, int inSize) +{ + (void)in; + return Lizard_saveDict(Lizard_stream, out, inSize); +} + +static int local_Lizard_compress_default_large(const char* in, char* out, int inSize) +{ + return Lizard_compress_MinLevel(in, out, inSize, Lizard_compressBound(inSize)); +} + +static int local_Lizard_compress_default_small(const char* in, char* out, int inSize) +{ + return Lizard_compress_MinLevel(in, out, inSize, Lizard_compressBound(inSize)-1); +} + +static int local_Lizard_compress_withState(const char* in, char* out, int inSize) +{ + return Lizard_compress_extState_MinLevel(Lizard_stream, in, out, inSize, Lizard_compressBound(inSize)); +} + +static int local_Lizard_compress_limitedOutput_withState(const char* in, char* out, int inSize) +{ + return Lizard_compress_extState_MinLevel(Lizard_stream, in, out, inSize, Lizard_compressBound(inSize)-1); +} + +static int local_Lizard_compress_continue(const char* in, char* out, int inSize) +{ + return Lizard_compress_continue(Lizard_stream, in, out, inSize, Lizard_compressBound(inSize)); +} + +static int local_Lizard_compress_limitedOutput_continue(const char* in, char* out, int inSize) +{ + return Lizard_compress_continue(Lizard_stream, in, out, inSize, Lizard_compressBound(inSize)-1); +} + + +/* HC compression functions */ +Lizard_stream_t* Lizard_streamPtr; +static void local_Lizard_resetStream(void) +{ + Lizard_streamPtr = Lizard_resetStream(Lizard_streamPtr, 0); +} + +static int local_Lizard_saveDictHC(const char* in, char* out, int inSize) +{ + (void)in; + return Lizard_saveDict(Lizard_streamPtr, out, inSize); +} + +static int local_Lizard_compress_extState(const char* in, char* out, int inSize) +{ + return Lizard_compress_extState(Lizard_streamPtr, in, out, inSize, Lizard_compressBound(inSize), 0); +} + +static int local_Lizard_compress_extState_limitedOutput(const char* in, char* out, int inSize) +{ + return Lizard_compress_extState(Lizard_streamPtr, in, out, inSize, Lizard_compressBound(inSize)-1, 0); +} + +static int local_Lizard_compress(const char* in, char* out, int inSize) +{ + return Lizard_compress(in, out, inSize, Lizard_compressBound(inSize), 0); +} + +static int local_Lizard_compress_limitedOutput(const char* in, char* out, int inSize) +{ + return Lizard_compress(in, out, inSize, Lizard_compressBound(inSize)-1, 0); +} + +static int local_Lizard_compressHC_continue(const char* in, char* out, int inSize) +{ + return Lizard_compress_continue(Lizard_streamPtr, in, out, inSize, Lizard_compressBound(inSize)); +} + +static int local_Lizard_compress_continue_limitedOutput(const char* in, char* out, int inSize) +{ + return Lizard_compress_continue(Lizard_streamPtr, in, out, inSize, Lizard_compressBound(inSize)-1); +} + + +/* decompression functions */ +static int local_Lizard_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize) +{ + (void)inSize; + Lizard_decompress_safe_usingDict(in, out, inSize, outSize, out - 65536, 65536); + return outSize; +} + +extern int Lizard_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const char* dict, int dictSize); + +static int local_Lizard_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize) +{ + (void)inSize; + Lizard_decompress_safe_forceExtDict(in, out, inSize, outSize, out - 65536, 65536); + return outSize; +} + +static int local_Lizard_decompress_safe_partial(const char* in, char* out, int inSize, int outSize) +{ + return Lizard_decompress_safe_partial(in, out, inSize, outSize - 5, outSize); +} + + +/* frame functions */ +static int local_LizardF_compressFrame(const char* in, char* out, int inSize) +{ + return (int)LizardF_compressFrame(out, 2*inSize + 16, in, inSize, NULL); +} + +static LizardF_decompressionContext_t g_dCtx; + +static int local_LizardF_decompress(const char* in, char* out, int inSize, int outSize) +{ + size_t srcSize = inSize; + size_t dstSize = outSize; + size_t result; + result = LizardF_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL); + if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame (%d)\n", (int)result); exit(8); } + if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect\n"); exit(9); } + return (int)dstSize; +} + + +#define NB_COMPRESSION_ALGORITHMS 100 +#define NB_DECOMPRESSION_ALGORITHMS 100 +int fullSpeedBench(const char** fileNamesTable, int nbFiles) +{ + int fileIdx=0; + + /* Init */ + { size_t const errorCode = LizardF_createDecompressionContext(&g_dCtx, LIZARDF_VERSION); + if (LizardF_isError(errorCode)) { DISPLAY("dctx allocation issue \n"); return 10; } } + + Lizard_streamPtr = Lizard_createStream(0); + if (!Lizard_streamPtr) { DISPLAY("Lizard_streamPtr allocation issue \n"); return 10; } + + Lizard_stream = Lizard_createStream_MinLevel(); + if (!Lizard_stream) { DISPLAY("Lizard_stream allocation issue \n"); return 10; } + + /* Loop for each fileName */ + while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) + DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); + + /* Allocation */ + chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)g_chunkSize)+1) * sizeof(struct chunkParameters)); + orig_buff = (char*) malloc(benchedSize); + nbChunks = (int) ((benchedSize + (g_chunkSize-1)) / g_chunkSize); + maxCompressedChunkSize = Lizard_compressBound(g_chunkSize); + compressedBuffSize = nbChunks * maxCompressedChunkSize; + compressed_buff = (char*)malloc((size_t)compressedBuffSize); + if(!chunkP || !orig_buff || !compressed_buff) { + DISPLAY("\nError: not enough memory!\n"); + fclose(inFile); + free(orig_buff); + free(compressed_buff); + free(chunkP); + return(12); + } + + /* Fill in src buffer */ + DISPLAY("Loading %s... \r", inFileName); + readSize = fread(orig_buff, 1, benchedSize, inFile); + fclose(inFile); + + if (readSize != benchedSize) { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(orig_buff); + free(compressed_buff); + free(chunkP); + return 13; + } + + /* Calculating input Checksum */ + crcOriginal = XXH32(orig_buff, benchedSize,0); + + + /* Bench */ + { int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb; + size_t cSize=0; + double ratio=0.; + + DISPLAY("\r%79s\r", ""); + DISPLAY(" %s : \n", inFileName); + + /* Bench Compression Algorithms */ + for (cAlgNb=0; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (g_compressionTest); cAlgNb++) { + const char* compressorName; + int (*compressionFunction)(const char*, char*, int); + void (*initFunction)(void) = NULL; + double bestTime = 100000000.; + + /* filter compressionAlgo only */ + if ((g_compressionAlgo != ALL_COMPRESSORS) && (g_compressionAlgo != cAlgNb)) continue; + + /* Init data chunks */ + { int i; + size_t remaining = benchedSize; + char* in = orig_buff; + char* out = compressed_buff; + nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); + for (i=0; i g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; + chunkP[i].compressedSize = 0; + } + } + + switch(cAlgNb) + { + case 0 : DISPLAY("Compression functions : \n"); continue; + case 1 : compressionFunction = local_Lizard_compress_default_large; compressorName = "Lizard_compress_MinLevel"; break; + case 2 : compressionFunction = local_Lizard_compress_default_small; compressorName = "Lizard_compress_MinLevel(small dst)"; break; + + case 10: compressionFunction = local_Lizard_compress; compressorName = "Lizard_compress"; break; + case 11: compressionFunction = local_Lizard_compress_limitedOutput; compressorName = "Lizard_compress limitedOutput"; break; + case 12 : compressionFunction = local_Lizard_compress_extState; compressorName = "Lizard_compress_extState"; break; + case 13: compressionFunction = local_Lizard_compress_extState_limitedOutput; compressorName = "Lizard_compress_extState limitedOutput"; break; + case 14: compressionFunction = local_Lizard_compressHC_continue; initFunction = local_Lizard_resetStream; compressorName = "Lizard_compress_continue"; break; + case 15: compressionFunction = local_Lizard_compress_continue_limitedOutput; initFunction = local_Lizard_resetStream; compressorName = "Lizard_compress_continue limitedOutput"; break; + case 30: compressionFunction = local_LizardF_compressFrame; compressorName = "LizardF_compressFrame"; + chunkP[0].origSize = (int)benchedSize; nbChunks=1; + break; + case 40: compressionFunction = local_Lizard_saveDict; compressorName = "Lizard_saveDict"; + Lizard_loadDict(Lizard_stream, chunkP[0].origBuffer, chunkP[0].origSize); + break; + case 41: compressionFunction = local_Lizard_saveDictHC; compressorName = "Lizard_saveDict"; + Lizard_loadDict(Lizard_streamPtr, chunkP[0].origBuffer, chunkP[0].origSize); + break; + case 16: compressionFunction = local_Lizard_compress_withState; compressorName = "Lizard_compress_extState_MinLevel(1)"; break; + case 17: compressionFunction = local_Lizard_compress_limitedOutput_withState; compressorName = "Lizard_compress_extState_MinLevel(1) limitedOutput"; break; + case 18: compressionFunction = local_Lizard_compress_continue; initFunction = local_Lizard_createStream; compressorName = "Lizard_compress_continue(1)"; break; + case 19: compressionFunction = local_Lizard_compress_limitedOutput_continue; initFunction = local_Lizard_createStream; compressorName = "Lizard_compress_continue(1) limitedOutput"; break; + case 60: DISPLAY("Obsolete compression functions : \n"); continue; + default : + continue; /* unknown ID : just skip */ + } + + for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { + double averageTime; + clock_t clockTime; + + PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize); + { size_t i; for (i=0; i%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000); + } + + if (ratio<100.) + DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000); + else + DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 100000); + } + + /* Prepare layout for decompression */ + /* Init data chunks */ + { int i; + size_t remaining = benchedSize; + char* in = orig_buff; + char* out = compressed_buff; + + nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); + for (i=0; i g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; + chunkP[i].compressedSize = 0; + } + } + for (chunkNb=0; chunkNb\r", loopNb, dName, (int)benchedSize); + + nb_loops = 0; + clockTime = clock(); + while(clock() == clockTime); + clockTime = clock(); + while(BMK_GetClockSpan(clockTime) < TIMELOOP) { + for (chunkNb=0; chunkNb %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000); + + /* CRC Checking */ + crcDecoded = XXH32(orig_buff, (int)benchedSize, 0); + if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); } + } + + DISPLAY("%2i-%-29.29s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000); + } + } + free(orig_buff); + free(compressed_buff); + free(chunkP); + } + + Lizard_freeStream(Lizard_stream); + Lizard_freeStream(Lizard_streamPtr); + LizardF_freeDecompressionContext(g_dCtx); + if (g_pause) { printf("press enter...\n"); (void)getchar(); } + + return 0; +} + + +static int usage(const char* exename) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); + DISPLAY( "Arguments :\n"); + DISPLAY( " -c : compression tests only\n"); + DISPLAY( " -d : decompression tests only\n"); + DISPLAY( " -H/-h : Help (this text + advanced options)\n"); + return 0; +} + +static int usage_advanced(void) +{ + DISPLAY( "\nAdvanced options :\n"); + DISPLAY( " -c# : test only compression function # [1-%i]\n", NB_COMPRESSION_ALGORITHMS); + DISPLAY( " -d# : test only decompression function # [1-%i]\n", NB_DECOMPRESSION_ALGORITHMS); + DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); + DISPLAY( " -B# : Block size [4-7](default : 7)\n"); + return 0; +} + +static int badusage(const char* exename) +{ + DISPLAY("Wrong parameters\n"); + usage(exename); + return 0; +} + +int main(int argc, const char** argv) +{ + int i, + filenamesStart=2; + const char* exename = argv[0]; + const char* input_filename=0; + + // Welcome message + DISPLAY(WELCOME_MESSAGE); + + if (argc<2) { badusage(exename); return 1; } + + for(i=1; i= '0') && (argument[1]<= '9')) { + g_compressionAlgo *= 10; + g_compressionAlgo += argument[1] - '0'; + argument++; + } + break; + + // Select decompression algorithm only + case 'd': + g_compressionTest = 0; + while ((argument[1]>= '0') && (argument[1]<= '9')) { + g_decompressionAlgo *= 10; + g_decompressionAlgo += argument[1] - '0'; + argument++; + } + break; + + // Display help on usage + case 'h' : + case 'H': usage(exename); usage_advanced(); return 0; + + // Modify Block Properties + case 'B': + while (argument[1]!=0) + switch(argument[1]) + { + case '4': + case '5': + case '6': + case '7': + { int B = argument[1] - '0'; + int S = 1 << (8 + 2*B); + BMK_setBlocksize(S); + argument++; + break; + } + case 'D': argument++; break; + default : goto _exit_blockProperties; + } +_exit_blockProperties: + break; + + // Modify Nb Iterations + case 'i': + if ((argument[1] >='0') && (argument[1] <='9')) { + int iters = argument[1] - '0'; + BMK_setNbIterations(iters); + argument++; + } + break; + + // Pause at the end (hidden option) + case 'p': BMK_setPause(); break; + + // Unknown command + default : badusage(exename); return 1; + } + } + continue; + } + + // first provided filename is input + if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } + + } + + // No input filename ==> Error + if(!input_filename) { badusage(exename); return 1; } + + return fullSpeedBench(argv+filenamesStart, argc-filenamesStart); + +} diff --git a/contrib/lizard/tests/fuzzer.c b/contrib/lizard/tests/fuzzer.c new file mode 100644 index 00000000000..90864dc6d02 --- /dev/null +++ b/contrib/lizard/tests/fuzzer.c @@ -0,0 +1,1086 @@ +/* + fuzzer.c - Fuzzer test tool for Lizard + Copyright (C) Yann Collet 2012-2016 + Copyright (C) Przemyslaw Skibinski 2016-2017 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - Lizard source repo : https://github.com/inikep/lizard +*/ + +/*-************************************ +* Compiler options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +# pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ +#endif + +/* S_ISREG & gettimeofday() are not supported by MSVC */ +#if defined(_MSC_VER) || defined(_WIN32) +# define FUZ_LEGACY_TIMER 1 +#endif + + +/*-************************************ +* Includes +**************************************/ +#include +#include /* fgets, sscanf */ +#include /* strcmp */ +#include /* clock_t, clock, CLOCKS_PER_SEC */ +#include "lizard_compress.h" /* LIZARD_VERSION_STRING */ +#include "lizard_decompress.h" +#include "lizard_common.h" +#define XXH_STATIC_LINKING_ONLY +#include "xxhash/xxhash.h" + + + +/*-************************************ +* Constants +**************************************/ +#define NB_ATTEMPTS (1<<16) +#define COMPRESSIBLE_NOISE_LENGTH (1 << 21) +#define FUZ_MAX_BLOCK_SIZE (1 << 17) +#define FUZ_MAX_DICT_SIZE (1 << 15) +#define FUZ_COMPRESSIBILITY_DEFAULT 60 +#define PRIME1 2654435761U +#define PRIME2 2246822519U +#define PRIME3 3266489917U + + +/*-*************************************** +* Macros +*****************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static int g_displayLevel = 2; +static const clock_t g_refreshRate = CLOCKS_PER_SEC * 25 / 100; +static clock_t g_time = 0; + + +/*-******************************************************* +* Fuzzer functions +*********************************************************/ +static clock_t FUZ_GetClockSpan(clock_t clockStart) +{ + return clock() - clockStart; /* works even if overflow; max span ~ 30mn */ +} + +static U32 FUZ_rotl32(U32 u32, U32 nbBits) +{ + return ((u32 << nbBits) | (u32 >> (32 - nbBits))); +} + +static U32 FUZ_rand(U32* src) +{ + U32 rand32 = *src; + rand32 *= PRIME1; + rand32 ^= PRIME2; + rand32 = FUZ_rotl32(rand32, 13); + *src = rand32; + return rand32; +} + + +#define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767) +#define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) +static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed) +{ + BYTE* BBuffer = (BYTE*)buffer; + size_t pos = 0; + U32 P32 = (U32)(32768 * proba); + + /* First Bytes */ + while (pos < 20) + BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); + + while (pos < bufferSize) + { + /* Select : Literal (noise) or copy (within 64K) */ + if (FUZ_RAND15BITS < P32) + { + /* Copy (within 64K) */ + size_t match, d; + size_t length = FUZ_RANDLENGTH + 4; + size_t offset = FUZ_RAND15BITS + 1; + while (offset > pos) offset >>= 1; + d = pos + length; + while (d > bufferSize) d = bufferSize; + match = pos - offset; + while (pos < d) BBuffer[pos++] = BBuffer[match++]; + } + else + { + /* Literal (noise) */ + size_t d; + size_t length = FUZ_RANDLENGTH; + d = pos + length; + if (d > bufferSize) d = bufferSize; + while (pos < d) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); + } + } +} + + +#define MAX_NB_BUFF_I134 150 +#define BLOCKSIZE_I134 (32 MB) +/*! FUZ_AddressOverflow() : +* Aggressively pushes memory allocation limits, +* and generates patterns which create address space overflow. +* only possible in 32-bits mode */ +static int FUZ_AddressOverflow(U32* seed) +{ + char* buffers[MAX_NB_BUFF_I134]; + int i, nbBuff=0; + int highAddress = 0; + + DISPLAY("Overflow tests : "); + + /* Only possible in 32-bits */ + /* if (sizeof(void*)==8) + { + DISPLAY("64 bits mode : no overflow \n"); + fflush(stdout); + return 0; + }*/ + + buffers[0] = (char*)malloc(BLOCKSIZE_I134); + buffers[1] = (char*)malloc(BLOCKSIZE_I134); + if ((!buffers[0]) || (!buffers[1])) { + DISPLAY("not enough memory for tests \n"); + return 0; + } + + for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++) { + DISPLAY("%3i \b\b\b\b", nbBuff); + buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134); + if (buffers[nbBuff]==NULL) goto _endOfTests; + + if (((size_t)buffers[nbBuff] > (size_t)0x80000000) && (!highAddress)) { + DISPLAY("high address detected : "); + fflush(stdout); + highAddress=1; + } + + { int const nbOf255 = 1 + (FUZ_rand(seed) % (BLOCKSIZE_I134-1)); + char* const input = buffers[nbBuff-1]; + char* output = buffers[nbBuff]; + int r; + BYTE cLevel = LIZARD_MIN_CLEVEL + (FUZ_rand(seed) % (1+LIZARD_MAX_CLEVEL-LIZARD_MIN_CLEVEL)); + for(i = 5; i < nbOf255; i++) input[i] = (char)0xff; + for(i = 5; i < nbOf255; i+=(FUZ_rand(seed) % 128)) input[i] = (BYTE)(FUZ_rand(seed)%256); + + input[0] = (char)cLevel; /* Compression Level */ + input[1] = (char)0xF0; /* Literal length overflow */ + input[2] = (char)0xFF; + input[3] = (char)0xFF; + input[4] = (char)0xFF; + r = Lizard_decompress_safe(input, output, nbOf255, BLOCKSIZE_I134); + if (r>0 && r0 && r0 && r0 && r g_refreshRate) | (g_displayLevel>=3)) { + g_time = clock(); + DISPLAY("\r%5u ", testNb); + if (g_displayLevel>=3) fflush(stdout); + } +} + + +/*! FUZ_findDiff() : +* find the first different byte between buff1 and buff2. +* presumes buff1 != buff2. +* presumes a difference exists before end of either buffer. +* Typically invoked after a checksum mismatch. +*/ +static void FUZ_findDiff(const void* buff1, const void* buff2) +{ + const BYTE* const b1 = (const BYTE*)buff1; + const BYTE* const b2 = (const BYTE*)buff2; + size_t i=0; + while (b1[i]==b2[i]) i++; + DISPLAY("Wrong Byte at position %u\n", (unsigned)i); +} + + +static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration_s) +{ + unsigned long long bytes = 0; + unsigned long long cbytes = 0; + unsigned long long hcbytes = 0; + unsigned long long ccbytes = 0; + void* CNBuffer; + char* compressedBuffer; + char* decodedBuffer; +# define FUZ_max LIZARD_COMPRESSBOUND(LEN) + int ret; + unsigned cycleNb; +# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ + printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } +# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); } + void* stateLizard = malloc(Lizard_sizeofState_MinLevel()); + void* stateLizardHC = malloc(Lizard_sizeofState(0)); + Lizard_stream_t* Lizarddict; + Lizard_stream_t* Lizard_streamHCPtr; + U32 crcOrig, crcCheck; + U32 coreRandState = seed; + U32 randState = coreRandState ^ PRIME3; + int result = 0; + clock_t const clockStart = clock(); + clock_t const clockDuration = (clock_t)duration_s * CLOCKS_PER_SEC; + + + /* init */ + Lizard_streamHCPtr = Lizard_createStream(0); + Lizarddict = Lizard_createStream_MinLevel(); + + /* Create compressible test buffer */ + CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); + compressedBuffer = (char*)malloc(Lizard_compressBound(FUZ_MAX_BLOCK_SIZE)); + decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); + + if (!stateLizard || !stateLizardHC || !Lizarddict || !Lizard_streamHCPtr || !CNBuffer || !compressedBuffer || !decodedBuffer) goto _output_error; + + FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); + + /* move to startCycle */ + for (cycleNb = 0; cycleNb < startCycle; cycleNb++) { + (void)FUZ_rand(&coreRandState); + + if (0) { /* some problems can be related to dictionary re-use; in this case, enable this loop */ + int dictSize, blockSize, blockStart; + char* dict; + char* block; + FUZ_displayUpdate(cycleNb); + randState = coreRandState ^ PRIME3; + blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; + blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); + dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; + if (dictSize > blockStart) dictSize = blockStart; + block = ((char*)CNBuffer) + blockStart; + dict = block - dictSize; + Lizard_loadDict(Lizarddict, dict, dictSize); + Lizard_compress_continue(Lizarddict, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); + Lizard_loadDict(Lizarddict, dict, dictSize); + Lizard_compress_continue(Lizarddict, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); + Lizard_loadDict(Lizarddict, dict, dictSize); + Lizard_compress_continue(Lizarddict, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); + } } + + /* Main test loop */ + for (cycleNb = startCycle; (cycleNb < nbCycles) || (FUZ_GetClockSpan(clockStart) < clockDuration) ; cycleNb++) { + U32 testNb = 0; + char* dict; + char* block; + int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize; + int blockContinueCompressedSize; + + FUZ_displayUpdate(cycleNb); + (void)FUZ_rand(&coreRandState); + randState = coreRandState ^ PRIME3; + + /* Select block to test */ + blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1; + blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); + dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; + if (dictSize > blockStart) dictSize = blockStart; + block = ((char*)CNBuffer) + blockStart; + dict = block - dictSize; + + /* Compression tests */ + + /* Test compression HC */ + FUZ_DISPLAYTEST; + ret = Lizard_compress(block, compressedBuffer, blockSize, Lizard_compressBound(blockSize), 0); + FUZ_CHECKTEST(ret==0, "Lizard_compress() failed"); + HCcompressedSize = ret; + + /* Test compression HC using external state */ + FUZ_DISPLAYTEST; + ret = Lizard_compress_extState(stateLizardHC, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize), 0); + FUZ_CHECKTEST(ret==0, "Lizard_compress_extState() failed"); + + /* Test compression using external state */ + FUZ_DISPLAYTEST; + ret = Lizard_compress_extState_MinLevel(stateLizard, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); + FUZ_CHECKTEST(ret==0, "Lizard_compress_extState_MinLevel(1) failed"); + + /* Test compression */ + FUZ_DISPLAYTEST; + ret = Lizard_compress_MinLevel(block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); + FUZ_CHECKTEST(ret==0, "Lizard_compress_MinLevel() failed"); + compressedSize = ret; + + /* Decompression tests */ + + crcOrig = XXH32(block, blockSize, 0); + + /* Test decoding with output size exactly what's necessary => must work */ + FUZ_DISPLAYTEST; + decodedBuffer[blockSize] = 0; + ret = Lizard_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); + FUZ_CHECKTEST(ret<0, "Lizard_decompress_safe failed despite sufficient space"); + FUZ_CHECKTEST(ret!=blockSize, "Lizard_decompress_safe did not regenerate original data"); + FUZ_CHECKTEST(decodedBuffer[blockSize], "Lizard_decompress_safe overrun specified output buffer size"); + crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "Lizard_decompress_safe corrupted decoded data"); + + // Test decoding with more than enough output size => must work + FUZ_DISPLAYTEST; + decodedBuffer[blockSize] = 0; + decodedBuffer[blockSize+1] = 0; + ret = Lizard_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1); + FUZ_CHECKTEST(ret<0, "Lizard_decompress_safe failed despite amply sufficient space"); + FUZ_CHECKTEST(ret!=blockSize, "Lizard_decompress_safe did not regenerate original data"); + //FUZ_CHECKTEST(decodedBuffer[blockSize], "Lizard_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ? + FUZ_CHECKTEST(decodedBuffer[blockSize+1], "Lizard_decompress_safe overrun specified output buffer size"); + crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "Lizard_decompress_safe corrupted decoded data"); + + // Test decoding with output size being one byte too short => must fail + FUZ_DISPLAYTEST; + decodedBuffer[blockSize-1] = 0; + ret = Lizard_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1); + FUZ_CHECKTEST(ret>=0, "Lizard_decompress_safe should have failed, due to Output Size being one byte too short"); + FUZ_CHECKTEST(decodedBuffer[blockSize-1], "Lizard_decompress_safe overrun specified output buffer size"); + + // Test decoding with output size being 10 bytes too short => must fail + FUZ_DISPLAYTEST; + if (blockSize>10) + { + decodedBuffer[blockSize-10] = 0; + ret = Lizard_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10); + FUZ_CHECKTEST(ret>=0, "Lizard_decompress_safe should have failed, due to Output Size being 10 bytes too short"); + FUZ_CHECKTEST(decodedBuffer[blockSize-10], "Lizard_decompress_safe overrun specified output buffer size"); + } + + // Test decoding with input size being one byte too short => must fail + FUZ_DISPLAYTEST; + ret = Lizard_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize); + FUZ_CHECKTEST(ret>=0, "Lizard_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize); + + // Test decoding with input size being one byte too large => must fail + FUZ_DISPLAYTEST; + decodedBuffer[blockSize] = 0; + compressedBuffer[compressedSize] = 0; /* valgrind */ + ret = Lizard_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize); + FUZ_CHECKTEST(ret>=0, "Lizard_decompress_safe should have failed, due to input size being too large"); + FUZ_CHECKTEST(decodedBuffer[blockSize], "Lizard_decompress_safe overrun specified output buffer size"); + + // Test partial decoding with target output size being max/2 => must work + FUZ_DISPLAYTEST; + ret = Lizard_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize); + FUZ_CHECKTEST(ret<0, "Lizard_decompress_safe_partial failed despite sufficient space"); + + // Test partial decoding with target output size being just below max => must work + FUZ_DISPLAYTEST; + ret = Lizard_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize); + FUZ_CHECKTEST(ret<0, "Lizard_decompress_safe_partial failed despite sufficient space"); + + /* Test Compression with limited output size */ + + /* Test compression with output size being exactly what's necessary (should work) */ + FUZ_DISPLAYTEST; + ret = Lizard_compress_MinLevel(block, compressedBuffer, blockSize, compressedSize); + FUZ_CHECKTEST(ret==0, "Lizard_compress_MinLevel() failed despite sufficient space"); + + /* Test compression with output size being exactly what's necessary and external state (should work) */ + FUZ_DISPLAYTEST; + ret = Lizard_compress_extState_MinLevel(stateLizard, block, compressedBuffer, blockSize, compressedSize); + FUZ_CHECKTEST(ret==0, "Lizard_compress_extState_MinLevel() failed despite sufficient space"); + + /* Test HC compression with output size being exactly what's necessary (should work) */ + FUZ_DISPLAYTEST; + ret = Lizard_compress(block, compressedBuffer, blockSize, HCcompressedSize, 0); + FUZ_CHECKTEST(ret==0, "Lizard_compress() (limitedOutput) failed despite sufficient space"); + + /* Test HC compression with output size being exactly what's necessary (should work) */ + FUZ_DISPLAYTEST; + ret = Lizard_compress_extState(stateLizardHC, block, compressedBuffer, blockSize, HCcompressedSize, 0); + FUZ_CHECKTEST(ret==0, "Lizard_compress_extState() failed despite sufficient space"); + + /* Test compression with missing bytes into output buffer => must fail */ + FUZ_DISPLAYTEST; + { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; + if (missingBytes >= compressedSize) missingBytes = compressedSize-1; + missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ + compressedBuffer[compressedSize-missingBytes] = 0; + ret = Lizard_compress_MinLevel(block, compressedBuffer, blockSize, compressedSize-missingBytes); + FUZ_CHECKTEST(ret, "Lizard_compress_MinLevel should have failed (output buffer too small by %i byte)", missingBytes); + FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "Lizard_compress_MinLevel overran output buffer ! (%i missingBytes)", missingBytes) + } + + /* Test HC compression with missing bytes into output buffer => must fail */ + FUZ_DISPLAYTEST; + { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; + if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; + missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ + compressedBuffer[HCcompressedSize-missingBytes] = 0; + ret = Lizard_compress(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, 0); + FUZ_CHECKTEST(ret, "Lizard_compress(limitedOutput) should have failed (output buffer too small by %i byte)", missingBytes); + FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "Lizard_compress overran output buffer ! (%i missingBytes)", missingBytes) + } + + + /*-******************/ + /* Dictionary tests */ + /*-******************/ + + /* Compress using dictionary */ + FUZ_DISPLAYTEST; + { Lizard_stream_t* Lizard_stream = Lizard_createStream_MinLevel(); + FUZ_CHECKTEST(Lizard_stream==NULL, "Lizard_createStream_MinLevel() allocation failed"); + Lizard_stream = Lizard_resetStream_MinLevel(Lizard_stream); + FUZ_CHECKTEST(Lizard_stream==NULL, "Lizard_resetStream_MinLevel() failed"); + Lizard_compress_continue (Lizard_stream, dict, compressedBuffer, dictSize, Lizard_compressBound(dictSize)); /* Just to fill hash tables */ + blockContinueCompressedSize = Lizard_compress_continue (Lizard_stream, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "Lizard_compress_continue failed"); + Lizard_freeStream(Lizard_stream); + } + + /* Compress using External dictionary */ + FUZ_DISPLAYTEST; + dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */ + if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; + Lizard_loadDict(Lizarddict, dict, dictSize); + blockContinueCompressedSize = Lizard_compress_continue(Lizarddict, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "Lizard_compress_continue failed"); + + FUZ_DISPLAYTEST; + Lizard_loadDict(Lizarddict, dict, dictSize); + ret = Lizard_compress_continue(Lizarddict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); + FUZ_CHECKTEST(ret>0, "Lizard_compress_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); + + FUZ_DISPLAYTEST; + Lizard_loadDict(Lizarddict, dict, dictSize); + ret = Lizard_compress_continue(Lizarddict, block, compressedBuffer, blockSize, blockContinueCompressedSize); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "Lizard_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(ret<=0, "Lizard_compress_continue should work : enough size available within output buffer"); + + FUZ_DISPLAYTEST; + decodedBuffer[blockSize] = 0; + ret = Lizard_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); + FUZ_CHECKTEST(ret!=blockSize, "2Lizard_decompress_safe_usingDict did not regenerate original data ret[%d]!=blockSize[%d]", (int)ret, (int)blockSize); + FUZ_CHECKTEST(decodedBuffer[blockSize], "Lizard_decompress_safe_usingDict overrun specified output buffer size") + crcCheck = XXH32(decodedBuffer, blockSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "Lizard_decompress_safe_usingDict corrupted decoded data"); + + FUZ_DISPLAYTEST; + decodedBuffer[blockSize-1] = 0; + ret = Lizard_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize); + FUZ_CHECKTEST(ret>=0, "Lizard_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); + FUZ_CHECKTEST(decodedBuffer[blockSize-1], "Lizard_decompress_safe_usingDict overrun specified output buffer size"); + + FUZ_DISPLAYTEST; + { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; + if ((U32)blockSize > missingBytes) { + decodedBuffer[blockSize-missingBytes] = 0; + ret = Lizard_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize); + FUZ_CHECKTEST(ret>=0, "Lizard_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes); + FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "Lizard_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize); + } } + + /* Compress HC using External dictionary */ + FUZ_DISPLAYTEST; + dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ + if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; + Lizard_streamHCPtr = Lizard_resetStream (Lizard_streamHCPtr, FUZ_rand(&randState) & 0x7); + FUZ_CHECKTEST(Lizard_streamHCPtr==NULL, "Lizard_resetStream failed"); + Lizard_loadDict(Lizard_streamHCPtr, dict, dictSize); + blockContinueCompressedSize = Lizard_compress_continue(Lizard_streamHCPtr, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "Lizard_compress_continue failed"); + + FUZ_DISPLAYTEST; + Lizard_loadDict(Lizard_streamHCPtr, dict, dictSize); + ret = Lizard_compress_continue(Lizard_streamHCPtr, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); + FUZ_CHECKTEST(ret>0, "Lizard_compress_continue using ExtDict should fail : one missing byte for output buffer"); + + FUZ_DISPLAYTEST; + Lizard_loadDict(Lizard_streamHCPtr, dict, dictSize); + ret = Lizard_compress_continue(Lizard_streamHCPtr, block, compressedBuffer, blockSize, blockContinueCompressedSize); + FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "Lizard_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); + FUZ_CHECKTEST(ret<=0, "Lizard_compress_continue should work : enough size available within output buffer"); + + FUZ_DISPLAYTEST; + decodedBuffer[blockSize] = 0; + ret = Lizard_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); + FUZ_CHECKTEST(ret!=blockSize, "3Lizard_decompress_safe_usingDict did not regenerate original data ret[%d]!=blockSize[%d]", (int)ret, (int)blockSize); + FUZ_CHECKTEST(decodedBuffer[blockSize], "Lizard_decompress_safe_usingDict overrun specified output buffer size") + crcCheck = XXH32(decodedBuffer, blockSize, 0); + if (crcCheck!=crcOrig) + FUZ_findDiff(block, decodedBuffer); + FUZ_CHECKTEST(crcCheck!=crcOrig, "Lizard_decompress_safe_usingDict corrupted decoded data"); + + /* ***** End of tests *** */ + /* Fill stats */ + bytes += blockSize; + cbytes += compressedSize; + hcbytes += HCcompressedSize; + ccbytes += blockContinueCompressedSize; + } + + if (nbCycles<=1) nbCycles = cycleNb; /* end by time */ + bytes += !bytes; /* avoid division by 0 */ + printf("\r%7u /%7u - ", cycleNb, nbCycles); + printf("all tests completed successfully \n"); + printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); + printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); + printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100); + + /* release memory */ + { +_exit: + free(CNBuffer); + free(compressedBuffer); + free(decodedBuffer); + free(stateLizard); + free(stateLizardHC); + Lizard_freeStream(Lizard_streamHCPtr); + Lizard_freeStream(Lizarddict); + return result; + +_output_error: + result = 1; + goto _exit; + } +} + + +#define testInputSize (192 KB) +#define testCompressedSize (128 KB) +#define ringBufferSize (8 KB) + +static void FUZ_unitTests(U32 seed) +{ + const unsigned testNb = 0; + const unsigned cycleNb= 0; + char testInput[testInputSize]; + char testCompressed[testCompressedSize]; + char testVerify[testInputSize]; + char ringBuffer[ringBufferSize]; + U32 randState = seed ^ PRIME3; + + /* Init */ + FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState); + + /* 32-bits address space overflow test */ + FUZ_AddressOverflow(&randState); + + /* Lizard streaming tests */ + { Lizard_stream_t* statePtr; + Lizard_stream_t* streamingState; + U64 crcOrig; + U64 crcNew; + int result; + + /* Allocation test */ + statePtr = Lizard_createStream_MinLevel(); + FUZ_CHECKTEST(statePtr==NULL, "Lizard_createStream_MinLevel() allocation failed"); + Lizard_freeStream(statePtr); + + streamingState = Lizard_createStream_MinLevel(); + FUZ_CHECKTEST(streamingState==NULL, "Lizard_createStream_MinLevel() allocation failed"); + + /* simple compression test */ + crcOrig = XXH64(testInput, testCompressedSize, 0); + streamingState = Lizard_resetStream_MinLevel(streamingState); + FUZ_CHECKTEST(streamingState==NULL, "Lizard_resetStream_MinLevel() failed"); + result = Lizard_compress_continue(streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "Lizard_compress_continue() compression failed"); + + result = Lizard_decompress_safe(testCompressed, testVerify, result, testCompressedSize); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "Lizard_decompress_safe() decompression failed Level 1 (result=%d testCompressedSize=%d)", (int)result, (int)testCompressedSize); + crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "Lizard_decompress_safe() decompression corruption"); + + /* ring buffer test */ + { XXH64_state_t xxhOrig; + XXH64_state_t xxhNew; + Lizard_streamDecode_t decodeState; + const U32 maxMessageSizeLog = 10; + const U32 maxMessageSizeMask = (1< ringBufferSize) rNext = 0; + if (dNext + messageSize > dBufferSize) dNext = 0; + } + } + Lizard_freeStream(streamingState); + } + + /* Lizard streaming tests */ + { Lizard_stream_t* streamPtr; + U64 crcOrig; + U64 crcNew; + int result; + + /* Allocation test */ + streamPtr = Lizard_createStream(0); + FUZ_CHECKTEST(streamPtr==NULL, "Lizard_createStream() allocation failed"); + + /* simple HC compression test */ + crcOrig = XXH64(testInput, testCompressedSize, 0); + streamPtr = Lizard_resetStream(streamPtr, 0); + FUZ_CHECKTEST(streamPtr==NULL, "Lizard_resetStream failed"); + result = Lizard_compress_continue(streamPtr, testInput, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "Lizard_compress_continue() compression failed"); + + result = Lizard_decompress_safe(testCompressed, testVerify, result, testCompressedSize); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "Lizard_decompress_safe() decompression failed Level 0 (result=%d testCompressedSize=%d)", (int)result, (int)testCompressedSize); + crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "Lizard_decompress_safe() decompression corruption"); + + /* simple dictionary HC compression test */ + crcOrig = XXH64(testInput + testInputSize - testCompressedSize, testCompressedSize, 0); + streamPtr = Lizard_resetStream(streamPtr, 0); + FUZ_CHECKTEST(streamPtr==NULL, "Lizard_resetStream failed"); + Lizard_loadDict(streamPtr, testInput, testInputSize - testCompressedSize); + result = Lizard_compress_continue(streamPtr, testInput + testInputSize - testCompressedSize, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "Lizard_compress_continue() dictionary compression failed : result = %i", result); + + result = Lizard_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, testInputSize - testCompressedSize); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "Lizard_decompress_safe() simple dictionary decompression test failed"); + crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "Lizard_decompress_safe() simple dictionary decompression test : corruption"); + + /* multiple HC compression test with dictionary */ + { int result1, result2; + int segSize = testCompressedSize / 2; + crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); + streamPtr = Lizard_resetStream(streamPtr, 0); + FUZ_CHECKTEST(streamPtr==NULL, "Lizard_resetStream failed"); + Lizard_loadDict(streamPtr, testInput, segSize); + result1 = Lizard_compress_continue(streamPtr, testInput + segSize, testCompressed, segSize, segSize -1); + FUZ_CHECKTEST(result1==0, "Lizard_compress_continue() dictionary compression failed : result = %i", result1); + result2 = Lizard_compress_continue(streamPtr, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1); + FUZ_CHECKTEST(result2==0, "Lizard_compress_continue() dictionary compression failed : result = %i", result2); + + result = Lizard_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize); + FUZ_CHECKTEST(result!=segSize, "Lizard_decompress_safe() dictionary decompression part 1 failed"); + result = Lizard_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); + FUZ_CHECKTEST(result!=segSize, "Lizard_decompress_safe() dictionary decompression part 2 failed"); + crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "Lizard_decompress_safe() dictionary decompression corruption"); + } + + /* remote dictionary HC compression test */ + crcOrig = XXH64(testInput + testInputSize - testCompressedSize, testCompressedSize, 0); + streamPtr = Lizard_resetStream(streamPtr, 0); + FUZ_CHECKTEST(streamPtr==NULL, "Lizard_resetStream failed"); + Lizard_loadDict(streamPtr, testInput, 32 KB); + result = Lizard_compress_continue(streamPtr, testInput + testInputSize - testCompressedSize, testCompressed, testCompressedSize, testCompressedSize-1); + FUZ_CHECKTEST(result==0, "Lizard_compress_continue() remote dictionary failed : result = %i", result); + + result = Lizard_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); + FUZ_CHECKTEST(result!=(int)testCompressedSize, "Lizard_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); + crcNew = XXH64(testVerify, testCompressedSize, 0); + FUZ_CHECKTEST(crcOrig!=crcNew, "Lizard_decompress_safe_usingDict() decompression corruption"); + + /* multiple HC compression with ext. dictionary */ + { XXH64_state_t crcOrigState; + XXH64_state_t crcNewState; + const char* dict = testInput + 3; + int dictSize = (FUZ_rand(&randState) & 8191); + char* dst = testVerify; + + size_t segStart = dictSize + 7; + int segSize = (FUZ_rand(&randState) & 8191); + int segNb = 1; + + streamPtr = Lizard_resetStream(streamPtr, 0); + FUZ_CHECKTEST(streamPtr==NULL, "Lizard_resetStream failed"); + Lizard_loadDict(streamPtr, dict, dictSize); + + XXH64_reset(&crcOrigState, 0); + XXH64_reset(&crcNewState, 0); + + while (segStart + segSize < testInputSize) { + XXH64_update(&crcOrigState, testInput + segStart, segSize); + crcOrig = XXH64_digest(&crcOrigState); + result = Lizard_compress_continue(streamPtr, testInput + segStart, testCompressed, segSize, Lizard_compressBound(segSize)); + FUZ_CHECKTEST(result==0, "Lizard_compress_continue() dictionary compression failed : result = %i", result); + + result = Lizard_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); + FUZ_CHECKTEST(result!=segSize, "Lizard_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); + XXH64_update(&crcNewState, dst, segSize); + crcNew = XXH64_digest(&crcNewState); + if (crcOrig!=crcNew) { + size_t c=0; + while (dst[c] == testInput[segStart+c]) c++; + DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)segSize); + } + FUZ_CHECKTEST(crcOrig!=crcNew, "Lizard_decompress_safe_usingDict() part %i corruption", segNb); + + dict = dst; + //dict = testInput + segStart; + dictSize = segSize; + + dst += segSize + 1; + segNb ++; + + segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1; + segSize = (FUZ_rand(&randState) & 8191); + } + } + + /* ring buffer test */ + { XXH64_state_t xxhOrig; + XXH64_state_t xxhNew; + Lizard_streamDecode_t decodeState; + const U32 maxMessageSizeLog = 10; + const U32 maxMessageSizeMask = (1< ringBufferSize) rNext = 0; + if (dNext + messageSize > dBufferSize) dNext = 0; + } + } + + /* small decoder-side ring buffer test */ + { XXH64_state_t xxhOrig; + XXH64_state_t xxhNew; + Lizard_streamDecode_t decodeState; + const U32 maxMessageSizeLog = 12; + const U32 maxMessageSizeMask = (1< dBufferSize) dNext = 0; + } + } + + Lizard_freeStream(streamPtr); + } + + printf("All unit tests completed successfully \n"); + return; +_output_error: + exit(1); +} + + +static int FUZ_usage(char* programName) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args]\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS); + DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); + DISPLAY( " -s# : Select seed (default:prompt user)\n"); + DISPLAY( " -t# : Select starting test number (default:0)\n"); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); + DISPLAY( " -v : verbose\n"); + DISPLAY( " -p : pause at the end\n"); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + + +int main(int argc, char** argv) +{ + U32 seed=0; + int seedset=0; + int argNb; + int nbTests = NB_ATTEMPTS; + int testNb = 0; + int proba = FUZ_COMPRESSIBILITY_DEFAULT; + int pause = 0; + char* programName = argv[0]; + U32 duration = 0; + + /* Check command line */ + for(argNb=1; argNb='0') && (*argument<='9')) { + nbTests *= 10; + nbTests += *argument - '0'; + argument++; + } + break; + + case 'T': + argument++; + nbTests = 0; duration = 0; + for (;;) { + switch(*argument) + { + case 'm': duration *= 60; argument++; continue; + case 's': + case 'n': argument++; continue; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': duration *= 10; duration += *argument++ - '0'; continue; + } + break; + } + break; + + case 's': + argument++; + seed=0; seedset=1; + while ((*argument>='0') && (*argument<='9')) { + seed *= 10; + seed += *argument - '0'; + argument++; + } + break; + + case 't': /* select starting test nb */ + argument++; + testNb=0; + while ((*argument>='0') && (*argument<='9')) { + testNb *= 10; + testNb += *argument - '0'; + argument++; + } + break; + + case 'P': /* change probability */ + argument++; + proba=0; + while ((*argument>='0') && (*argument<='9')) { + proba *= 10; + proba += *argument - '0'; + argument++; + } + if (proba<0) proba=0; + if (proba>100) proba=100; + break; + default: ; + } + } + } + } + + printf("Starting Lizard fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LIZARD_VERSION_STRING); + + if (!seedset) { + time_t const t = time(NULL); + U32 const h = XXH32(&t, sizeof(t), 1); + seed = h % 10000; + } + printf("Seed = %u\n", seed); + + if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); + + if (testNb==0) FUZ_unitTests(seed); + + if (nbTests<=0) nbTests=1; + + { int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); + if (pause) { + DISPLAY("press enter ... \n"); + (void)getchar(); + } + return result; + } +} diff --git a/contrib/lizard/visual/.gitignore b/contrib/lizard/visual/.gitignore new file mode 100644 index 00000000000..538cd1a21c8 --- /dev/null +++ b/contrib/lizard/visual/.gitignore @@ -0,0 +1,11 @@ +# Visual C++ +.vs/ +*Copy +*.db +*.opensdf +*.sdf +*.suo +*.user +*.opendb + +VS2010/bin/ diff --git a/contrib/lizard/visual/README.md b/contrib/lizard/visual/README.md new file mode 100644 index 00000000000..527e4be1874 --- /dev/null +++ b/contrib/lizard/visual/README.md @@ -0,0 +1,53 @@ +Projects for various integrated development environments (IDE) +============================================================== + +#### Included projects + +The following projects are included with the lizard distribution: +- `VS2010` - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015) + + +#### How to compile lizard with Visual Studio + +1. Install Visual Studio e.g. VS 2015 Community Edition (it's free). +2. Download the latest version of lizard from https://github.com/inikep/lizard/releases +3. Decompress ZIP archive. +4. Go to decompressed directory then to `visual` then `VS2010` and open `lizard.sln` +5. Visual Studio will ask about converting VS2010 project to VS2015 and you should agree. +6. Change `Debug` to `Release` and if you have 64-bit Windows change also `Win32` to `x64`. +7. Press F7 on keyboard or select `BUILD` from the menu bar and choose `Build Solution`. +8. If compilation will be fine a compiled executable will be in `visual\VS2010\bin\x64_Release\lizard.exe` + + +#### Projects available within lizard.sln + +The Visual Studio solution file `lizard.sln` contains many projects that will be compiled to the +`visual\VS2010\bin\$(Platform)_$(Configuration)` directory. For example `lizard` set to `x64` and +`Release` will be compiled to `visual\VS2010\bin\x64_Release\lizard.exe`. The solution file contains the +following projects: + +- `lizard` : Command Line Utility, supporting gzip-like arguments +- `datagen` : Synthetic and parametrable data generator, for tests +- `frametest` : Test tool that checks lizard_frame integrity on target platform +- `fullbench` : Precisely measure speed for each lizard inner functions +- `fuzzer` : Test tool, to check lizard integrity on target platform +- `liblizard` : A static Lizard library compiled to `liblizard_static.lib` +- `liblizard-dll` : A dynamic Lizard library (DLL) compiled to `liblizard.dll` with the import library `liblizard.lib` +- `fullbench-dll` : The fullbench program compiled with the import library; the executable requires Lizard DLL + + +#### Using Lizard DLL with Microsoft Visual C++ project + +The header files `lib\lizard.h`, `lib\lizardhc.h`, `lib\lizard_frame.h` and the import library +`visual\VS2010\bin\$(Platform)_$(Configuration)\liblizard.lib` are required to compile a +project using Visual C++. + +1. The path to header files should be added to `Additional Include Directories` that can + be found in Project Properties of Visual Studio IDE in the `C/C++` Property Pages on the `General` page. +2. The import library has to be added to `Additional Dependencies` that can + be found in Project Properties in the `Linker` Property Pages on the `Input` page. + If one will provide only the name `liblizard.lib` without a full path to the library + then the directory has to be added to `Linker\General\Additional Library Directories`. + +The compiled executable will require Lizard DLL which is available at +`visual\VS2010\bin\$(Platform)_$(Configuration)\liblizard.dll`. diff --git a/contrib/lizard/visual/VS2010/datagen/datagen.vcxproj b/contrib/lizard/visual/VS2010/datagen/datagen.vcxproj new file mode 100644 index 00000000000..7f7581d01b8 --- /dev/null +++ b/contrib/lizard/visual/VS2010/datagen/datagen.vcxproj @@ -0,0 +1,165 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {D745AE2F-596A-403A-9B91-81A8C6779243} + Win32Proj + datagen + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + Application + true + Unicode + + + Application + true + Unicode + + + Application + false + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + /analyze:stacksize295252 %(AdditionalOptions) + + + Console + true + true + true + + + + + + + + + + + + + \ No newline at end of file diff --git a/contrib/lizard/visual/VS2010/frametest/frametest.vcxproj b/contrib/lizard/visual/VS2010/frametest/frametest.vcxproj new file mode 100644 index 00000000000..b753d651be1 --- /dev/null +++ b/contrib/lizard/visual/VS2010/frametest/frametest.vcxproj @@ -0,0 +1,177 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7} + Win32Proj + frametest + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + Application + true + Unicode + + + Application + true + Unicode + + + Application + false + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + true + /analyze:stacksize295252 %(AdditionalOptions) + + + Console + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/contrib/lizard/visual/VS2010/fullbench/fullbench.vcxproj b/contrib/lizard/visual/VS2010/fullbench/fullbench.vcxproj new file mode 100644 index 00000000000..f05bb298055 --- /dev/null +++ b/contrib/lizard/visual/VS2010/fullbench/fullbench.vcxproj @@ -0,0 +1,175 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E} + Win32Proj + fullbench + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + Application + true + Unicode + + + Application + true + Unicode + + + Application + false + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + + + Console + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/contrib/lizard/visual/VS2010/fuzzer/fuzzer.vcxproj b/contrib/lizard/visual/VS2010/fuzzer/fuzzer.vcxproj new file mode 100644 index 00000000000..7fa0fec022f --- /dev/null +++ b/contrib/lizard/visual/VS2010/fuzzer/fuzzer.vcxproj @@ -0,0 +1,174 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {18B9F1A7-9C66-4352-898B-30804DADE0FD} + Win32Proj + fuzzer + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + Application + true + Unicode + + + Application + true + Unicode + + + Application + false + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + + + Console + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + + + Console + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + true + /analyze:stacksize295252 %(AdditionalOptions) + + + Console + true + true + true + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.rc b/contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.rc new file mode 100644 index 00000000000..79110101f3b --- /dev/null +++ b/contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.rc @@ -0,0 +1,51 @@ +// Microsoft Visual C++ generated resource script. +// + +#include "lizard_compress.h" /* LIZARD_VERSION_STRING */ +#define APSTUDIO_READONLY_SYMBOLS +#include "verrsrc.h" +#undef APSTUDIO_READONLY_SYMBOLS + + +#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) +LANGUAGE 9, 1 + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// + +VS_VERSION_INFO VERSIONINFO + FILEVERSION LIZARD_VERSION_MAJOR,LIZARD_VERSION_MINOR,LIZARD_VERSION_RELEASE,0 + PRODUCTVERSION LIZARD_VERSION_MAJOR,LIZARD_VERSION_MINOR,LIZARD_VERSION_RELEASE,0 + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG + FILEFLAGS VS_FF_DEBUG +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS_NT_WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904B0" + BEGIN + VALUE "CompanyName", "Yann Collet, Przemyslaw Skibinski" + VALUE "FileDescription", "Extremely fast compression" + VALUE "FileVersion", LIZARD_VERSION_STRING + VALUE "InternalName", "liblizard.dll" + VALUE "LegalCopyright", "Copyright (C) 2013-2017, Yann Collet, Przemyslaw Skibinski" + VALUE "OriginalFilename", "liblizard.dll" + VALUE "ProductName", "Lizard" + VALUE "ProductVersion", LIZARD_VERSION_STRING + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0409, 1200 + END +END + +#endif diff --git a/contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.vcxproj b/contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.vcxproj new file mode 100644 index 00000000000..09cdaa16180 --- /dev/null +++ b/contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.vcxproj @@ -0,0 +1,179 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {00000000-3E71-41B3-BF68-4A7BDD8A5476} + Win32Proj + liblizard-dll + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + DynamicLibrary + true + Unicode + + + DynamicLibrary + true + Unicode + + + DynamicLibrary + false + true + Unicode + + + DynamicLibrary + false + true + Unicode + + + + + + + + + + + + + + + + + + + true + liblizard + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + liblizard + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + liblizard + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + liblizard + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) + true + false + + + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + + + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) + false + false + + + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) + false + true + /analyze:stacksize295252 %(AdditionalOptions) + + + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/contrib/lizard/visual/VS2010/liblizard/liblizard.vcxproj b/contrib/lizard/visual/VS2010/liblizard/liblizard.vcxproj new file mode 100644 index 00000000000..15b74523b96 --- /dev/null +++ b/contrib/lizard/visual/VS2010/liblizard/liblizard.vcxproj @@ -0,0 +1,176 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476} + Win32Proj + liblizard + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + StaticLibrary + true + Unicode + + + StaticLibrary + true + Unicode + + + StaticLibrary + false + true + Unicode + + + StaticLibrary + false + true + Unicode + + + + + + + + + + + + + + + + + + + true + liblizard_static + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + liblizard_static + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + liblizard_static + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + liblizard_static + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) + true + false + + + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + + + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) + false + false + + + true + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) + false + true + /analyze:stacksize295252 %(AdditionalOptions) + + + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/contrib/lizard/visual/VS2010/lizard.sln b/contrib/lizard/visual/VS2010/lizard.sln new file mode 100644 index 00000000000..72d68335c1e --- /dev/null +++ b/contrib/lizard/visual/VS2010/lizard.sln @@ -0,0 +1,86 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Express 2012 for Windows Desktop +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lizard", "lizard\lizard.vcxproj", "{E30329AC-0057-4FE0-8FDA-7F650D398C4C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblizard", "liblizard\liblizard.vcxproj", "{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblizard-dll", "liblizard-dll\liblizard-dll.vcxproj", "{00000000-3E71-41B3-BF68-4A7BDD8A5476}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{18B9F1A7-9C66-4352-898B-30804DADE0FD}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "frametest", "frametest\frametest.vcxproj", "{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{D745AE2F-596A-403A-9B91-81A8C6779243}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.ActiveCfg = Debug|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.Build.0 = Debug|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.ActiveCfg = Debug|x64 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.Build.0 = Debug|x64 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.ActiveCfg = Release|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.Build.0 = Release|Win32 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.ActiveCfg = Release|x64 + {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.Build.0 = Release|x64 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.ActiveCfg = Debug|Win32 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.Build.0 = Debug|Win32 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.ActiveCfg = Debug|x64 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.Build.0 = Debug|x64 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.ActiveCfg = Release|Win32 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.Build.0 = Release|Win32 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.ActiveCfg = Release|x64 + {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.Build.0 = Release|x64 + {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.ActiveCfg = Debug|Win32 + {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.Build.0 = Debug|Win32 + {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.ActiveCfg = Debug|x64 + {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.Build.0 = Debug|x64 + {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.ActiveCfg = Release|Win32 + {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.Build.0 = Release|Win32 + {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.ActiveCfg = Release|x64 + {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.Build.0 = Release|x64 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.ActiveCfg = Debug|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.Build.0 = Debug|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.ActiveCfg = Debug|x64 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.Build.0 = Debug|x64 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.ActiveCfg = Release|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.Build.0 = Release|Win32 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.ActiveCfg = Release|x64 + {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.Build.0 = Release|x64 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.ActiveCfg = Debug|Win32 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.Build.0 = Debug|Win32 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.ActiveCfg = Debug|x64 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.Build.0 = Debug|x64 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.ActiveCfg = Release|Win32 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.Build.0 = Release|Win32 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.ActiveCfg = Release|x64 + {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.Build.0 = Release|x64 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.ActiveCfg = Debug|Win32 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.Build.0 = Debug|Win32 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.ActiveCfg = Debug|x64 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.Build.0 = Debug|x64 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.ActiveCfg = Release|Win32 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.Build.0 = Release|Win32 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.ActiveCfg = Release|x64 + {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.Build.0 = Release|x64 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.ActiveCfg = Debug|Win32 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.Build.0 = Debug|Win32 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.ActiveCfg = Debug|x64 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.Build.0 = Debug|x64 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.ActiveCfg = Release|Win32 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.Build.0 = Release|Win32 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.ActiveCfg = Release|x64 + {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/contrib/lizard/visual/VS2010/lizard/lizard.rc b/contrib/lizard/visual/VS2010/lizard/lizard.rc new file mode 100644 index 00000000000..76c3c7f947e --- /dev/null +++ b/contrib/lizard/visual/VS2010/lizard/lizard.rc @@ -0,0 +1,51 @@ +// Microsoft Visual C++ generated resource script. +// + +#include "lizard_compress.h" /* LIZARD_VERSION_STRING */ +#define APSTUDIO_READONLY_SYMBOLS +#include "verrsrc.h" +#undef APSTUDIO_READONLY_SYMBOLS + + +#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) +LANGUAGE 9, 1 + +///////////////////////////////////////////////////////////////////////////// +// +// Version +// + +VS_VERSION_INFO VERSIONINFO + FILEVERSION LIZARD_VERSION_MAJOR,LIZARD_VERSION_MINOR,LIZARD_VERSION_RELEASE,0 + PRODUCTVERSION LIZARD_VERSION_MAJOR,LIZARD_VERSION_MINOR,LIZARD_VERSION_RELEASE,0 + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG + FILEFLAGS VS_FF_DEBUG +#else + FILEFLAGS 0x0L +#endif + FILEOS VOS_NT_WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE VFT2_UNKNOWN +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904B0" + BEGIN + VALUE "CompanyName", "Yann Collet, Przemyslaw Skibinski" + VALUE "FileDescription", "Extremely fast compression" + VALUE "FileVersion", LIZARD_VERSION_STRING + VALUE "InternalName", "lizard.exe" + VALUE "LegalCopyright", "Copyright (C) 2013-2017, Yann Collet, Przemyslaw Skibinski" + VALUE "OriginalFilename", "lizard.exe" + VALUE "ProductName", "Lizard" + VALUE "ProductVersion", LIZARD_VERSION_STRING + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0409, 1200 + END +END + +#endif diff --git a/contrib/lizard/visual/VS2010/lizard/lizard.vcxproj b/contrib/lizard/visual/VS2010/lizard/lizard.vcxproj new file mode 100644 index 00000000000..98be4d599db --- /dev/null +++ b/contrib/lizard/visual/VS2010/lizard/lizard.vcxproj @@ -0,0 +1,190 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {E30329AC-0057-4FE0-8FDA-7F650D398C4C} + Win32Proj + lizard + $(SolutionDir)bin\$(Platform)_$(Configuration)\ + $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ + + + + Application + true + MultiByte + + + Application + true + MultiByte + + + Application + false + true + MultiByte + + + Application + false + true + MultiByte + + + + + + + + + + + + + + + + + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + true + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + + + false + $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); + true + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + + + Console + true + setargv.obj;%(AdditionalDependencies) + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + true + /analyze:stacksize295252 %(AdditionalOptions) + + + Console + true + setargv.obj;%(AdditionalDependencies) + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + false + + + Console + true + setargv.obj;%(AdditionalDependencies) + true + true + + + + + Level4 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + false + true + /analyze:stacksize295252 %(AdditionalOptions) + + + Console + true + setargv.obj;%(AdditionalDependencies) + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/contrib/lzsse/.gitignore b/contrib/lzsse/.gitignore new file mode 100644 index 00000000000..379a8f7f570 --- /dev/null +++ b/contrib/lzsse/.gitignore @@ -0,0 +1,4 @@ +bin/* +gmake/* +vs2013/* +vs2015/* \ No newline at end of file diff --git a/contrib/lzsse/.travis.yml b/contrib/lzsse/.travis.yml new file mode 100644 index 00000000000..68bb59ef0b9 --- /dev/null +++ b/contrib/lzsse/.travis.yml @@ -0,0 +1,185 @@ +language: c +sudo: false +branches: + except: + - /^(wip\/)?(appveyor|msvc|mingw|windows)(\-.+)?$/ +matrix: + include: + ### + ## Linux builds using various versions of GCC. + ### + - env: C_COMPILER=gcc-7 + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - premake4 + packages: + - gcc-7 + - g++-7 + - premake4 + # - env: C_COMPILER=gcc-6 + # addons: + # apt: + # sources: + # - ubuntu-toolchain-r-test + # - premake4 + # packages: + # - gcc-6 + # - g++-6 + # - premake4 + - env: C_COMPILER=gcc-5 + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - premake4 + packages: + - gcc-5 + - g++-5 + - premake4 + # - env: C_COMPILER=gcc-4.9 + # addons: + # apt: + # sources: + # - ubuntu-toolchain-r-test + # - premake4 + # packages: + # - gcc-4.9 + # - g++-4.9 + # - premake4 + - env: C_COMPILER=gcc-4.8 + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - premake4 + packages: + - gcc-4.8 + - g++-4.8 + - premake4 + # - env: C_COMPILER=gcc-4.7 + # addons: + # apt: + # sources: + # - ubuntu-toolchain-r-test + # - premake4 + # packages: + # - gcc-4.7 + # - g++-4.7 + # - premake4 + - env: C_COMPILER=gcc-4.6 + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - premake4 + packages: + - gcc-4.6 + - g++-4.6 + - premake4 + # - os: linux + # env: C_COMPILER=gcc-4.5 + # addons: + # apt: + # sources: + # - ubuntu-toolchain-r-test + # - premake4 + # packages: + # - gcc-4.5 + # - g++-4.5 + # - premake4 + - env: C_COMPILER=gcc-4.4 + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - premake4 + packages: + - gcc-4.4 + - g++-4.4 + - premake4 + + ### + ## clang on Linux + ### + - env: C_COMPILER=clang-3.9 + addons: + apt: + sources: + - llvm-toolchain-precise-3.9 + - ubuntu-toolchain-r-test + - premake4 + packages: + - clang-3.9 + - premake4 + # - env: C_COMPILER=clang-3.8 + # addons: + # apt: + # sources: + # - llvm-toolchain-precise-3.8 + # - ubuntu-toolchain-r-test + # - premake4 + # packages: + # - clang-3.8 + # - premake4 + - env: C_COMPILER=clang-3.7 + addons: + apt: + sources: + - llvm-toolchain-precise-3.7 + - ubuntu-toolchain-r-test + - premake4 + packages: + - clang-3.7 + - premake4 + # - env: C_COMPILER=clang-3.6 + # addons: + # apt: + # sources: + # - llvm-toolchain-precise-3.6 + # - ubuntu-toolchain-r-test + # - premake4 + # packages: + # - clang-3.6 + # - premake4 + - env: C_COMPILER=clang-3.5 + addons: + apt: + sources: + - llvm-toolchain-precise-3.5 + - ubuntu-toolchain-r-test + - premake4 + packages: + - clang-3.5 + - premake4 + + ### + ## PGI + ### + - env: C_COMPILER=pgcc ENABLE_OPENMP=y + addons: + apt: + sources: + - premake4 + packages: + - premake4 + + ### + ## OS X + ### + # - os: osx + +before_install: +### +## If we use the matrix to set CC/CXX Travis, overwrites the values, +## so instead we use C/CXX_COMPILER, then copy the values to CC/CXX +## here (after Travis has set CC/CXX). +### +- if [ "${C_COMPILER}" = "pgcc" ]; then wget -q -O /dev/stdout 'https://raw.githubusercontent.com/nemequ/pgi-travis/master/install-pgi.sh' | /bin/sh; fi +- if [ -n "${C_COMPILER}" ]; then export CC="${C_COMPILER}"; fi +- premake4 gmake + +script: +- cd gmake && make + diff --git a/contrib/lzsse/LICENSE b/contrib/lzsse/LICENSE new file mode 100644 index 00000000000..d13dc62210f --- /dev/null +++ b/contrib/lzsse/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2016, Conor Stokes +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/lzsse/README.md b/contrib/lzsse/README.md new file mode 100644 index 00000000000..6b777eecb52 --- /dev/null +++ b/contrib/lzsse/README.md @@ -0,0 +1,15 @@ +# LZSSE +[LZSS](https://en.wikipedia.org/wiki/Lempel%E2%80%93Ziv%E2%80%93Storer%E2%80%93Szymanski) designed for a branchless SSE decompression implementation. + +Three variants: +- LZSSE2, for high compression files with small literal runs. +- LZSSE4, for a more balanced mix of literals and matches. +- LZSSE8, for lower compression data with longer runs of matches. + +All three variants have an optimal parser implementation, which uses a quite strong match finder (very similar to LzFind) combined with a Storer-Szymanski style parse. LZSSE4 and LZSSE8 have "fast" compressor implementations, which use a simple hash table based matching and a greedy parse. + +Currently LZSSE8 is the recommended variant to use in the general case, as it generally performs well in most cases (and you have the option of both optimal parse and fast compression). LZSSE2 is recommended if you are only using text, especially heavily compressible text, but is slow/doesn't compress as well on less compressible data and binaries. + +The code is approaching production readiness and LZSSE2 and LZSSE8 have received a reasonable amount of testing. + +See these blog posts [An LZ Codec Designed for SSE Decompression](http://conorstokes.github.io/compression/2016/02/15/an-LZ-codec-designed-for-SSE-decompression) and [Compressor Improvements and LZSSE2 vs LZSSE8](http://conorstokes.github.io/compression/2016/02/24/compressor-improvements-and-lzsse2-vs-lzsse8) for a description of how the compression algorithm and implementation function. There are also benchmarks, but these may not be upto date (in particular the figures in the initial blog post no longer represent compression performance). diff --git a/contrib/lzsse/example/main.cpp b/contrib/lzsse/example/main.cpp new file mode 100644 index 00000000000..9287f683c7d --- /dev/null +++ b/contrib/lzsse/example/main.cpp @@ -0,0 +1,389 @@ +// LZSSE.cpp : Defines the entry point for the console application. +// +#define _CRT_SECURE_NO_WARNINGS 1 + +#include +#include +#include +#include +#include "../lzsse2/lzsse2.h" +#include "../lzsse4/lzsse4.h" +#include "../lzsse8/lzsse8.h" + +static const uint32_t MAGIC_NUMBER = 0x28F19732; + +void DisplayUsage() +{ + printf( "Usage:\n" ); + printf( " lzsse [args] input_file output_file\n" ); + printf( "\n" ); + printf( "Arguments:\n" ); + printf( " -2 Compress in lzsse2 mode (default)\n" ); + printf( " -4 Compress in lzsse4 mode\n" ); + printf( " -8 Compress in lzsse8 mode\n" ); + printf( " -f Optimal parse (default)\n" ); + printf( " -o Fast parse (not available for lzsse2)\n" ); + printf( " -d Decompress\n" ); + printf( " -lN Compression level for optimal parse, where N is 1 to 17 (default 16)\n" ); + printf( " -bN Block size in KiB, default 131,072\n" ); + printf( "\n" ); +} + +static size_t CompressorFastLZSSE4( LZSSE4_FastParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength, unsigned int ) +{ + return LZSSE4_CompressFast( state, input, inputLength, output, outputLength ); +} + +static size_t CompressorFastLZSSE8( LZSSE8_FastParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength, unsigned int ) +{ + return LZSSE8_CompressFast( state, input, inputLength, output, outputLength ); +} + +template +void Compress( FILE* inputFile, FILE* outputFile, uint64_t blockSize, uint8_t mode, unsigned int level, State* state, size_t (*compressor)( State*, const void*, size_t, void*, size_t, unsigned int ) ) +{ + if ( state == nullptr ) + { + printf( "Couldn't allocate parse state\n" ); + exit( 1 ); + } + + if ( fwrite( &MAGIC_NUMBER, sizeof( uint32_t ), 1, outputFile ) == 0 ) + { + printf( "Couldn't write magic number\n" ); + exit( 1 ); + } + + if ( fwrite( &mode, sizeof( uint8_t ), 1, outputFile ) == 0 ) + { + printf( "Couldn't write stream type\n" ); + exit( 1 ); + } + + if ( fwrite( &blockSize, sizeof( uint64_t ), 1, outputFile ) == 0 ) + { + printf( "Couldn't write block size\n" ); + exit( 1 ); + } + + size_t typedBlockSize = static_cast< size_t >( blockSize ); + uint8_t* inputBuffer = reinterpret_cast< uint8_t* >( malloc( typedBlockSize ) ); + uint8_t* outputBuffer = reinterpret_cast< uint8_t* >( malloc( typedBlockSize ) ); + + if ( inputBuffer == nullptr || outputBuffer == nullptr ) + { + printf( "Couldn't allocate buffer memory\n" ); + exit( 1 ); + } + + for ( ;; ) + { + size_t readSize = fread( inputBuffer, 1, blockSize, inputFile ); + + if ( readSize == 0 ) + { + break; + } + + size_t compressedSize = compressor( state, inputBuffer, readSize, outputBuffer, typedBlockSize, level ); + + if ( compressedSize == 0 ) + { + printf( "Compression function failed\n" ); + exit( 1 ); + } + + uint32_t compressedLength = static_cast< uint32_t >( compressedSize ); + uint32_t uncompressedLength = static_cast< uint32_t >( readSize ); + + if ( fwrite( &uncompressedLength, sizeof( uint32_t ), 1, outputFile ) < 1 ) + { + break; + } + + if ( fwrite( &compressedLength, sizeof( uint32_t ), 1, outputFile ) < 1 ) + { + printf( "Error writing compressed length from block\n" ); + exit( 1 ); + } + + if ( fwrite( outputBuffer, 1, compressedLength, outputFile ) != compressedLength ) + { + printf( "Error writing block\n" ); + exit( 1 ); + } + } + + free( inputBuffer ); + free( outputBuffer ); +} + +void Decompress( FILE* inputFile, FILE* outputFile ) +{ + uint32_t magicNumber; + uint64_t blockSize = 128 * 1024 * 1024; + + if ( fread( &magicNumber, sizeof( uint32_t ), 1, inputFile ) < 1 || magicNumber != MAGIC_NUMBER ) + { + printf( "Couldn't read magic number, or magic number incorrect\n" ); + exit( 1 ); + } + + uint8_t streamType; + + if ( fread( &streamType, sizeof( uint8_t ), 1, inputFile ) < 1 ) + { + printf( "Couldn't read stream type\n" ); + exit( 1 ); + } + + if ( fread( &blockSize, sizeof( uint64_t ), 1, inputFile ) < 1 ) + { + printf( "Couldn't read block size\n" ); + exit( 1 ); + } + + uint8_t* inputBuffer = reinterpret_cast< uint8_t* >( malloc( static_cast< size_t >( blockSize ) ) ); + uint8_t* outputBuffer = reinterpret_cast< uint8_t* >( malloc( static_cast< size_t >( blockSize ) ) ); + + if ( inputBuffer == nullptr || outputBuffer == nullptr ) + { + printf( "Couldn't allocate buffer memory\n" ); + exit( 1 ); + } + + size_t( *decompressor )( const void*, size_t, void*, size_t ); + + switch ( streamType ) + { + case 2: + + decompressor = LZSSE2_Decompress; + break; + + case 4: + + decompressor = LZSSE4_Decompress; + break; + + case 8: + + decompressor = LZSSE8_Decompress; + break; + + default: + + printf( "Invalid stream type\n" ); + exit( 1 ); + + } + + memset( inputBuffer, 0, blockSize ); + memset( outputBuffer, 0, blockSize ); + + for ( ;; ) + { + uint32_t compressedLength; + uint32_t uncompressedLength; + + if ( fread( &uncompressedLength, sizeof( uint32_t ), 1, inputFile ) < 1 ) + { + break; + } + + if ( fread( &compressedLength, sizeof( uint32_t ), 1, inputFile ) < 1 ) + { + printf( "Error reading compressed length from block\n" ); + exit( 1 ); + } + + if ( fread( inputBuffer, 1, compressedLength, inputFile ) != compressedLength ) + { + printf( "Error reading block\n" ); + exit( 1 ); + } + + size_t decompressedSize = 0; + + decompressedSize = + decompressor( inputBuffer, + compressedLength, + outputBuffer, + uncompressedLength ); + + if ( decompressedSize != size_t( uncompressedLength ) ) + { + printf( "Error in decompression stream\n" ); + exit( 1 ); + } + + if ( fwrite( outputBuffer, 1, uncompressedLength, outputFile ) != uncompressedLength ) + { + printf( "Couldn't write block to output file\n" ); + exit( 1 ); + } + } + + free( inputBuffer ); + free( outputBuffer ); +} + +int main( int argc, const char** argv ) +{ + bool decompression = false; + bool optimal = true; + uint64_t blockSize = 128 * 1024 * 1024; + uint8_t mode = 2; + unsigned int level = 16; + + if ( argc < 3 ) + { + DisplayUsage(); + exit( 1 ); + } + + for ( int argIndex = 1; argIndex < argc - 2; ++argIndex ) + { + const char* arg = argv[ argIndex ]; + + if ( arg[ 0 ] == '-' ) + { + switch ( arg[ 1 ] ) + { + case 'd': + + decompression = true; + break; + + case '2': + + mode = 2; + break; + + case '4': + + mode = 4; + break; + + case '8': + + mode = 8; + break; + + case 'l': + + level = static_cast< unsigned int >( strtoul( arg + 2, nullptr, 10 ) ); + break; + + case 'b': + + blockSize = strtoull( arg + 2, nullptr, 10 ) * 1024; + break; + + case 'o': + + optimal = true; + break; + + case 'f': + + optimal = false; + break; + + } + } + } + + FILE* inputFile = fopen( argv[ argc - 2 ], "rb" ); + + if ( inputFile == nullptr ) + { + perror( argv[ argc - 2 ] ); + exit( 1 ); + } + + FILE* outputFile = fopen( argv[ argc - 1 ], "wb+" ); + + if ( outputFile == nullptr ) + { + perror( argv[ argc - 2 ] ); + exit( 1 ); + } + + if ( decompression ) + { + Decompress( inputFile, outputFile ); + } + else + { + switch ( mode ) + { + case 2: + { + LZSSE2_OptimalParseState* state = LZSSE2_MakeOptimalParseState( static_cast< size_t >( blockSize ) ); + + Compress( inputFile, outputFile, blockSize, mode, level, state, LZSSE2_CompressOptimalParse ); + + LZSSE2_FreeOptimalParseState( state ); + + break; + } + + case 4: + { + if ( optimal ) + { + LZSSE4_OptimalParseState* state = LZSSE4_MakeOptimalParseState( static_cast( blockSize ) ); + + Compress( inputFile, outputFile, blockSize, mode, level, state, LZSSE4_CompressOptimalParse ); + + LZSSE4_FreeOptimalParseState( state ); + } + else + { + LZSSE4_FastParseState* state = LZSSE4_MakeFastParseState(); + + Compress( inputFile, outputFile, blockSize, mode, level, state, CompressorFastLZSSE4 ); + + LZSSE4_FreeFastParseState( state ); + } + + break; + } + + case 8: + { + if ( optimal ) + { + LZSSE8_OptimalParseState* state = LZSSE8_MakeOptimalParseState( static_cast( blockSize ) ); + + Compress( inputFile, outputFile, blockSize, mode, level, state, LZSSE8_CompressOptimalParse ); + + LZSSE8_FreeOptimalParseState( state ); + } + else + { + LZSSE8_FastParseState* state = LZSSE8_MakeFastParseState(); + + Compress( inputFile, outputFile, blockSize, mode, level, state, CompressorFastLZSSE8 ); + + LZSSE8_FreeFastParseState( state ); + } + + break; + } + + default: + + printf( "Invalid stream type\n" ); + exit( 1 ); + + } + } + + fclose( inputFile ); + fclose( outputFile ); + + return 0; +} + diff --git a/contrib/lzsse/lzsse2/lzsse2.cpp b/contrib/lzsse/lzsse2/lzsse2.cpp new file mode 100644 index 00000000000..d521a84553a --- /dev/null +++ b/contrib/lzsse/lzsse2/lzsse2.cpp @@ -0,0 +1,1080 @@ +/* +Copyright (c) 2016, Conor Stokes +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include +#include +#include +#include + +#include "lzsse2_platform.h" +#include "lzsse2.h" + +#pragma warning ( disable : 4127 ) + +namespace +{ + // Constants - most of these should not be changed without corresponding code changes because it will break many things in unpredictable ways. + const uint32_t WINDOW_BITS = 16; + const uint32_t MIN_MATCH_LENGTH = 3; + const uint32_t LZ_WINDOW_SIZE = 1 << WINDOW_BITS; + const uint32_t LZ_WINDOW_MASK = LZ_WINDOW_SIZE - 1; + const uint32_t OPTIMAL_HASH_BITS = 20; + const uint32_t OPTIMAL_BUCKETS_COUNT = 1 << OPTIMAL_HASH_BITS; + const uint32_t OPTIMAL_HASH_MASK = OPTIMAL_BUCKETS_COUNT - 1; + const uint32_t MIN_COMPRESSION_SIZE = 32; + const uint32_t END_PADDING_LITERALS = 16; + const int32_t NO_MATCH = -1; + const int32_t EMPTY_NODE = -1; + const uint32_t MIN_LITERAL_COUNT = 8; + const uint32_t HASH_MULTIPLIER = 4013; + const uint32_t HASH_REMOVAL_MULTIPLIER = HASH_MULTIPLIER * HASH_MULTIPLIER; + const uint32_t CONTROL_BITS = 4; + const uint32_t LITERAL_BITS = 8; + const uint32_t OFFSET_BITS = 16; + const uint32_t BASE_MATCH_BITS = OFFSET_BITS + CONTROL_BITS; + const uint32_t SINGLE_LITERAL_COST = CONTROL_BITS + LITERAL_BITS; + const uint32_t DOUBLE_LITERAL_COST = SINGLE_LITERAL_COST + LITERAL_BITS; + const uint32_t EXTENDED_MATCH_BOUND = ( 1 << CONTROL_BITS ) - 1; + const uint32_t CONTROL_BLOCK_SIZE = sizeof( __m128i ); + const uint32_t CONTROLS_PER_BLOCK = 32; + const uint32_t LITERALS_PER_CONTROL = 2; + const uint32_t MAX_INPUT_PER_CONTROL = 2; + const size_t OUTPUT_BUFFER_SAFE = EXTENDED_MATCH_BOUND * CONTROLS_PER_BLOCK; + const size_t INPUT_BUFFER_SAFE = MAX_INPUT_PER_CONTROL * CONTROLS_PER_BLOCK; + const uint16_t INITIAL_OFFSET = MIN_MATCH_LENGTH; + const size_t SKIP_MATCH_LENGTH = 128; + const uint32_t NO_SKIP_LEVEL = 17; +} + +struct Arrival +{ + size_t cost; + int32_t from; + int32_t to; + uint16_t offset; +}; + +struct TreeNode +{ + int32_t children[ 2 ]; +}; + +struct LZSSE2_OptimalParseState +{ + // Note, we should really replace this with a BST, hash chaining works but is *slooooooooooooooow* for optimal parse. + int32_t roots[ OPTIMAL_BUCKETS_COUNT ]; + + TreeNode window[ LZ_WINDOW_SIZE ]; + + Arrival* arrivals; + + size_t bufferSize; +}; + + +LZSSE2_OptimalParseState* LZSSE2_MakeOptimalParseState( size_t bufferSize ) +{ + if ( bufferSize > 0 && ( SIZE_MAX / sizeof( Arrival ) ) < bufferSize ) + { + return nullptr; + } + + LZSSE2_OptimalParseState* result = reinterpret_cast< LZSSE2_OptimalParseState* >( ::malloc( sizeof( LZSSE2_OptimalParseState ) ) ); + + result->bufferSize = bufferSize; + + if ( result != nullptr ) + { + result->arrivals = reinterpret_cast< Arrival* >( ::malloc( sizeof( Arrival ) * bufferSize ) ); + + if ( result->arrivals == nullptr ) + { + LZSSE2_FreeOptimalParseState( result ); + + result = nullptr; + } + } + + return result; +} + + +void LZSSE2_FreeOptimalParseState( LZSSE2_OptimalParseState* toFree ) +{ + ::free( toFree->arrivals ); + + toFree->arrivals = nullptr; + + ::free( toFree ); +} + + +inline uint32_t CalculateHash( const uint8_t* inputCursor ) +{ + return ( uint32_t( inputCursor[ 0 ] ) * HASH_MULTIPLIER * HASH_MULTIPLIER + uint32_t( inputCursor[ 1 ] ) * HASH_MULTIPLIER + uint32_t( inputCursor[ 2 ] ) ) & OPTIMAL_HASH_MASK; +} + + +struct Match +{ + size_t length; + int32_t position; + uint16_t offset; +}; + + +inline Match SearchAndUpdateFinder( LZSSE2_OptimalParseState& state, const uint8_t* input, const uint8_t* inputCursor, const uint8_t* inputEnd, uint32_t cutOff ) +{ + Match result; + + int32_t position = static_cast( inputCursor - input ); + + result.position = NO_MATCH; + result.length = MIN_MATCH_LENGTH; + result.offset = 0; + + size_t lengthToEnd = inputEnd - inputCursor; + int32_t lastPosition = position - ( LZ_WINDOW_SIZE - 1 ); + uint32_t hash = CalculateHash( inputCursor ); + + lastPosition = lastPosition > 0 ? lastPosition : 0; + + int32_t treeCursor = state.roots[ hash ]; + + state.roots[ hash ] = position; + + int32_t* left = &state.window[ position & LZ_WINDOW_MASK ].children[ 1 ]; + int32_t* right = &state.window[ position & LZ_WINDOW_MASK ].children[ 0 ]; + size_t leftLength = 0; + size_t rightLength = 0; + + for ( ;; ) + { + if ( cutOff-- == 0 || treeCursor < lastPosition ) + { + *left = *right = EMPTY_NODE; + break; + } + + TreeNode& currentNode = state.window[ treeCursor & LZ_WINDOW_MASK ]; + const uint8_t* key = input + treeCursor; + size_t matchLength = leftLength < rightLength ? leftLength : rightLength; + + uint16_t matchOffset = static_cast( position - treeCursor ); + size_t maxLength = matchOffset <= ( EXTENDED_MATCH_BOUND + 1 ) && matchOffset < lengthToEnd ? matchOffset : lengthToEnd; + + while ( matchLength < lengthToEnd ) + { + __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); + __m128i match16 = _mm_loadu_si128( reinterpret_cast( key + matchLength ) ); + + unsigned long matchBytes; + + _BitScanForward( &matchBytes, ( static_cast( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); + + matchLength += matchBytes; + + if ( matchBytes != 16 ) + { + break; + } + } + + matchLength = matchLength < lengthToEnd ? matchLength : lengthToEnd; + + size_t truncatedMatchLength = matchLength < maxLength ? matchLength : maxLength; + + if ( truncatedMatchLength >= result.length ) + { + result.length = truncatedMatchLength; + result.offset = matchOffset; + result.position = treeCursor; + } + + if ( matchLength == lengthToEnd ) + { + *left = currentNode.children[ 1 ]; + *right = currentNode.children[ 0 ]; + break; + } + + if ( inputCursor[ matchLength ] < key[ matchLength ] || ( matchLength == lengthToEnd ) ) + { + *left = treeCursor; + left = currentNode.children; + treeCursor = *left; + leftLength = matchLength; + } + else + { + *right = treeCursor; + right = currentNode.children + 1; + treeCursor = *right; + rightLength = matchLength; + } + } + + // Special RLE overlapping match case, the LzFind style match above doesn't work very well with our + // restriction of overlapping matches having offsets of at least 16. + // Suffix array seems like a better option to handling this. + { + // Note, we're detecting long RLE here, but if we have an offset too close, we'll sacrifice a fair + // amount of decompression performance to load-hit-stores. + int32_t matchPosition = position - ( sizeof( __m128i ) * 2 ); + + if ( matchPosition >= 0 ) + { + uint16_t matchOffset = static_cast( position - matchPosition ); + const uint8_t* key = input + matchPosition; + size_t matchLength = 0; + + while ( matchLength < lengthToEnd ) + { + __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); + __m128i match16 = _mm_loadu_si128( reinterpret_cast( key + matchLength ) ); + + unsigned long matchBytes; + + _BitScanForward( &matchBytes, ( static_cast( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); + + matchLength += matchBytes; + + if ( matchBytes != 16 ) + { + break; + } + + } + + matchLength = matchLength < lengthToEnd ? matchLength : lengthToEnd; + + if ( matchLength >= result.length ) + { + result.length = matchLength; + result.offset = matchOffset; + result.position = matchPosition; + } + } + } + + return result; +} + + +size_t LZSSE2_CompressOptimalParse( LZSSE2_OptimalParseState* state, const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength, unsigned int level ) +{ + if ( outputLength < inputLength || state->bufferSize < inputLength ) + { + // error case, output buffer not large enough. + return 0; + } + + const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); + uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); + + if ( inputLength < MIN_COMPRESSION_SIZE ) + { + memcpy( output, input, inputLength ); + + return inputLength; + } + + const uint8_t* inputCursor = input; + const uint8_t* inputEnd = input + inputLength; + Arrival* arrivalWatermark = state->arrivals; + Arrival* arrival = state->arrivals; + uint32_t cutOff = 1 << level; + + for ( int32_t* rootCursor = state->roots, *end = rootCursor + OPTIMAL_BUCKETS_COUNT; rootCursor < end; rootCursor += 4 ) + { + rootCursor[ 0 ] = EMPTY_NODE; + rootCursor[ 1 ] = EMPTY_NODE; + rootCursor[ 2 ] = EMPTY_NODE; + rootCursor[ 3 ] = EMPTY_NODE; + } + + for ( uint32_t where = 0; where < MIN_MATCH_LENGTH; ++where ) + { + /*Match dummy = */ SearchAndUpdateFinder( *state, input, inputCursor, inputEnd - END_PADDING_LITERALS, cutOff ); + + ++inputCursor; + } + + arrival->cost = LITERAL_BITS * MIN_MATCH_LENGTH; + arrival->from = -1; + arrival->offset = 0; + + // loop through each character and project forward the matches at that character to calculate the cheapest + // path of arrival for each individual character. + for ( const uint8_t* earlyEnd = inputEnd - END_PADDING_LITERALS; inputCursor < earlyEnd; ++inputCursor, ++arrival ) + { + uint32_t lengthToEnd = static_cast< uint32_t >( earlyEnd - inputCursor ); + int32_t currentPosition = static_cast< int32_t >( inputCursor - input ); + Arrival* literalFirst = arrival + 1; + Arrival* literalSecond = arrival + 2; + size_t arrivalCost = arrival->cost; + + // NOTE - we currently assume only 2 literals filled in here, because the minimum match length is 3. + // If we wanted to go with a higher minimum match length, we would need to fill in more literals before hand. + // Also, because there is a maximum of 2 literals per control block assumed. + + // project forward the cost of a single literal + if ( literalFirst > arrivalWatermark || literalFirst->cost > ( arrival->cost + SINGLE_LITERAL_COST ) ) + { + literalFirst->cost = arrival->cost + SINGLE_LITERAL_COST; + literalFirst->from = currentPosition; + literalFirst->offset = 0; + + arrivalWatermark = literalFirst > arrivalWatermark ? literalFirst : arrivalWatermark; + } + + // project forward the cost of two literals + if ( lengthToEnd > 1 ) + { + if ( literalSecond > arrivalWatermark || literalFirst->cost > ( arrival->cost + DOUBLE_LITERAL_COST ) ) + { + literalSecond->cost = arrival->cost + DOUBLE_LITERAL_COST; + literalSecond->from = currentPosition; + literalSecond->offset = 0; + + arrivalWatermark = literalSecond > arrivalWatermark ? literalSecond : arrivalWatermark; + } + } + else + { + continue; + } + + Match match = SearchAndUpdateFinder( *state, input, inputCursor, earlyEnd, cutOff ); + + if ( match.position != NO_MATCH ) + { + for ( size_t matchedLength = MIN_MATCH_LENGTH, end = match.length + 1; matchedLength < end; ++matchedLength ) + { + Arrival* matchArrival = arrival + matchedLength; + size_t matchCost = arrivalCost + BASE_MATCH_BITS; + + if ( matchedLength > EXTENDED_MATCH_BOUND ) + { + matchCost += ( ( matchedLength - 1 ) / EXTENDED_MATCH_BOUND ) * CONTROL_BITS; + } + + if ( matchArrival > arrivalWatermark || matchArrival->cost > matchCost ) + { + matchArrival->cost = matchCost; + matchArrival->from = currentPosition; + matchArrival->offset = match.offset; + + arrivalWatermark = matchArrival > arrivalWatermark ? matchArrival : arrivalWatermark; + } + } + + if ( match.length > SKIP_MATCH_LENGTH && level < NO_SKIP_LEVEL ) + { + arrival += match.length - LITERALS_PER_CONTROL; + inputCursor += match.length - LITERALS_PER_CONTROL; + } + } + } + + // If this would cost more to encode than it would if it were just literals, encode it with no control blocks, + // just literals + if ( ( arrivalWatermark->cost + END_PADDING_LITERALS * LITERAL_BITS + CONTROLS_PER_BLOCK * CONTROL_BITS ) > ( inputLength * LITERAL_BITS ) ) + { + memcpy( output, input, inputLength ); + + return inputLength; + } + + Arrival* previousPathNode; + + // now trace the actual optimal parse path back, connecting the nodes in the other direction. + for ( const Arrival* pathNode = arrivalWatermark; pathNode->from > 0; pathNode = previousPathNode ) + { + previousPathNode = state->arrivals + ( pathNode->from - MIN_MATCH_LENGTH ); + + previousPathNode->to = static_cast( ( pathNode - state->arrivals ) + MIN_MATCH_LENGTH ); + } + + uint8_t* outputCursor = output; + + memcpy( outputCursor, input, MIN_MATCH_LENGTH ); + + outputCursor += MIN_MATCH_LENGTH; + + uint8_t* currentControlBlock = outputCursor; + uint32_t currentControlCount = 0; + uint32_t totalControlCount = 0; + + outputCursor += CONTROL_BLOCK_SIZE; + + Arrival* nextPathNode; + + size_t totalPathLength = MIN_MATCH_LENGTH; + uint16_t previousOffset = INITIAL_OFFSET; + + bool lastControlIsNop = false; + + // Now walk forwards again and actually write out the data. + for ( const Arrival* pathNode = state->arrivals; pathNode < arrivalWatermark; pathNode = nextPathNode ) + { + int32_t currentPosition = static_cast< int32_t >( ( pathNode - state->arrivals ) + MIN_MATCH_LENGTH ); + + nextPathNode = state->arrivals + ( pathNode->to - MIN_MATCH_LENGTH ); + + size_t pathDistance = nextPathNode - pathNode; + + totalPathLength += pathDistance; + + lastControlIsNop = false; + + if ( pathDistance < MIN_MATCH_LENGTH ) + { + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + currentControlCount = 0; + } + + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + ( static_cast( pathDistance ) - 1 ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + ( static_cast< uint8_t >( pathDistance ) - 1 ) << CONTROL_BITS; + } + + // output the literals. + for ( int32_t where = 0; where < pathDistance; ++where ) + { + const uint8_t* currentInput = input + currentPosition + where; + + outputCursor[ where ] = *currentInput ^ *( currentInput - previousOffset ); + } + + outputCursor += pathDistance; + + ++totalControlCount; + ++currentControlCount; + } + else + { + size_t toEncode = pathDistance - 1; //note, we always subtract one here, because the first control block of the match encodes this way + + // make sure the control block for the first part of the match has been allocated + // note, this is idempontent if we have not actually incremented the control count and we try this again. + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + currentControlCount = 0; + } + + // output the offset (after control block containing the first control nibble for this match). + *reinterpret_cast< uint16_t* >( outputCursor ) = nextPathNode->offset ^ previousOffset; + + previousOffset = nextPathNode->offset; + + outputCursor += sizeof( uint16_t ); + + for ( ;; ) + { + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + currentControlCount = 0; + } + + if ( toEncode >= EXTENDED_MATCH_BOUND ) + { + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + static_cast( EXTENDED_MATCH_BOUND ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + static_cast< uint8_t >( EXTENDED_MATCH_BOUND ) << CONTROL_BITS; + } + + toEncode -= EXTENDED_MATCH_BOUND; + + ++totalControlCount; + ++currentControlCount; + + } + else + { + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + static_cast( toEncode ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + static_cast< uint8_t >( toEncode ) << CONTROL_BITS; + } + + if ( toEncode == 0 && currentControlCount == 0 ) + { + lastControlIsNop = true; + } + + ++totalControlCount; + ++currentControlCount; + + break; + } + } + } + } + + if ( lastControlIsNop ) + { + outputCursor -= CONTROL_BLOCK_SIZE; + } + + size_t remainingLiterals = ( input + inputLength ) - inputCursor; + + // copy remaining literals + memcpy( outputCursor, inputCursor, remainingLiterals ); + + outputCursor += remainingLiterals; + + return outputCursor - output; +} + + +size_t LZSSE2_Decompress( const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength ) +{ + const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); + uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); + + // Length it not work compressing, just copy initial values + if ( outputLength == inputLength ) + { + memcpy( output, input, outputLength ); + + return inputLength; + } + + const uint8_t* inputCursor = input; + uint8_t* outputCursor = output; + + // The offset starts off as the minimum match length. We actually need it at least two + // characters back because we need them to be set to xor out the literals from the match data. + size_t offset = INITIAL_OFFSET; + __m128i previousCarryHi = _mm_setzero_si128(); + + *( outputCursor++ ) = *( inputCursor++ ); + *( outputCursor++ ) = *( inputCursor++ ); + *( outputCursor++ ) = *( inputCursor++ ); + + // What these macros do: + // Decode a single literal run or match run for a single control nibble. + // How they do it: + // - Read the *unaligned* input (in the case of LZSSE-F - twice, for LZSSE-O we read once) - one goes into an SSE register, + // because it could either be literals or an offset (or nothing at all). The low byte of streamBytesRead controls how much we advance + // the input cursor. + // - Used a contived set of casts to sign extend the "read offset" control mask and then use it to mask the input word, + // which is then xor'd against the offset, for a "branchless" conditional move into the offset which + // has been carried over from the previous literal/match block. Note, this ends up doing better than a cmov on most + // modern processors. But we need to pre-xor the input offset. + // - We then load the match data from output buffer (offset back from the current output point). Unconditional load here. + // - We broadcast the "from literal" control mask from the current least significant byte of the SSE register using a shuffle epi-8 + // - We mask the literals with that SSE register wide mask. + // - The literals have been pre-xor'd with the data read in as match data, so we use an xor to branchlessly choose between the two. + // In this case, it ends up a better option than a blendv on most processors. + // - Store the block. We store all 16 bytes of the SSE register (due to some constraints in the format of the data, we won't + // go past the end of the buffer), but we may overlap this. + // - bytesOut controls how much we advance the output cursor. + // - We use 8 bit shifts to advance all the controls up to the next byte. There is some variable sized register trickery that + // x86/x64 is great for as long as we don't anger the register renamer. + +#define DECODE_STEP( HILO, CHECKMATCH, CHECKBUFFERS ) \ + { \ + \ + uint64_t inputWord = *reinterpret_cast( inputCursor ); \ + __m128i literals = _mm_cvtsi64_si128( inputWord ); \ + \ + offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ + \ + readOffsetHalf##HILO >>= 8; \ + \ + const uint8_t* matchPointer = outputCursor - offset; \ + \ + if ( CHECKMATCH && matchPointer < output ) \ + goto MATCH_UNDERFLOW; \ + \ + __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ + __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ + \ + literals = _mm_and_si128( fromLiteral, literals ); \ + \ + fromLiteral##HILO = _mm_srli_si128( fromLiteral##HILO, 1 ); \ + \ + __m128i toStore = _mm_xor_si128( matchData, literals ); \ + \ + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ + \ + outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ + inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ + \ + bytesOutHalf##HILO >>= 8; \ + streamBytesReadHalf##HILO >>= 8; \ + \ + if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ + goto BUFFER_END; \ + } + +#define DECODE_STEP_HALF( HILO, CHECKMATCH, CHECKBUFFERS ) \ + { \ + size_t inputWord = *reinterpret_cast< const uint16_t* >( inputCursor ); \ + __m128i literals = _mm_cvtsi64_si128( inputWord ); \ + \ + offset ^= static_cast< size_t >( static_cast< ptrdiff_t >( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ + \ + const uint8_t* matchPointer = outputCursor - offset; \ + \ + if ( CHECKMATCH && matchPointer < output ) \ + goto MATCH_UNDERFLOW; \ + \ + __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ + __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ + \ + literals = _mm_and_si128( fromLiteral, literals ); \ + \ + fromLiteral##HILO = _mm_srli_si128( fromLiteral##HILO, 1 ); \ + \ + __m128i toStore = _mm_xor_si128( matchData, literals ); \ + \ + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ + \ + outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ + inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ + \ + if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ + goto BUFFER_END; \ + } + +#define DECODE_STEP_END( HILO, CHECKMATCH, CHECKBUFFERS ) \ + { \ + size_t inputWord = *reinterpret_cast< const uint16_t* >( inputCursor ); \ + __m128i literals = _mm_cvtsi64_si128( inputWord ); \ + \ + offset ^= static_cast< size_t >( static_cast< ptrdiff_t >( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ + \ + const uint8_t* matchPointer = outputCursor - offset; \ + \ + if ( CHECKMATCH && matchPointer < output ) \ + goto MATCH_UNDERFLOW; \ + \ + __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ + __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ + \ + literals = _mm_and_si128( fromLiteral, literals ); \ + \ + __m128i toStore = _mm_xor_si128( matchData, literals ); \ + \ + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ + \ + outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ + inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ + \ + if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ + goto BUFFER_END; \ + } + +#define DECODE_STEP_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP( Lo, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP( Hi, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_HALF_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_HALF( Lo, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_HALF_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_HALF( Hi, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_END_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_END( Lo, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_END_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_END( Hi, CHECKMATCH, CHECKBUFFERS ) + + __m128i nibbleMask = _mm_set1_epi8( 0xF ); + __m128i literalsPerControl = _mm_set1_epi8( LITERALS_PER_CONTROL ); + + // Note, we use this block here because it allows the "fake" inputEarlyEnd/outputEarlyEnd not to cause register spills + // in the decompression loops. And yes, that did actually happen. + { +#pragma warning ( push ) +#pragma warning ( disable : 4101 ) + + const uint8_t* inputEarlyEnd; //= ( input + inputLength ) - END_PADDING_LITERALS; + uint8_t* outputEarlyEnd;// = ( output + outputLength ) - END_PADDING_LITERALS; + +#pragma warning ( pop ) + + // "Safe" ends to the buffer, before the input/output cursors hit these, we can loop without overflow checks. + const uint8_t* inputSafeEnd = ( input + inputLength ) - INPUT_BUFFER_SAFE; + uint8_t* outputSafeEnd = ( output + outputLength ) - OUTPUT_BUFFER_SAFE; + + // Decoding loop with offset output buffer underflow test, but no buffer overflow tests, assumed to end at a safe distance + // from overflows + while ( ( outputCursor - output ) < LZ_WINDOW_SIZE && outputCursor < outputSafeEnd && inputCursor < inputSafeEnd ) + { + // load the control block + __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); + + // split the control block into high and low nibbles. + __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); + __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); + + // Here we are testing if the runs will be literals or matches. Note that if the carries are set from the previous operation + // this will essentially be ignored later on. + __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); + __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); + + // Work out the carry for the low nibbles (which will be used with the high controls to put them into + // match without offset read mode). + __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); + + // The carry for the high nibbles is used with the low controls, but needs one byte from the previous iteration. We save + // the calculated carry to use that byte next iteration. + __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); + __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); + + previousCarryHi = carryHi; + + // I want 128 set bits please. + __m128i allSet = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); + + // Calcualting the bytes to output to the stream. Basically, we are subtracting negative one from the control value if the + // carry is not set. This works because the masks produced by comparisons are the equivalent to negative one, which + // make this a conditional increment. + __m128i bytesOutLo = _mm_sub_epi8( controlLo, _mm_xor_si128( shiftedCarryHi, allSet ) ); + __m128i bytesOutHi = _mm_sub_epi8( controlHi, _mm_xor_si128( carryLo, allSet ) ); + + // Calculate the number of bytes to read per control. + // In the case the carry is set, no bytes. Otherwise, the offset size (2 bytes) for matches or the number of output bytes for literals. + __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_min_epi8( literalsPerControl, bytesOutLo ) ); + __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_min_epi8( literalsPerControl, bytesOutHi ) ); + + // Masks to read the offset (or keep the previous one) - set in the case that this is not a literal and the carry is not set + __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), allSet ); + __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), allSet ); + + // Masks whether we are reading literals - set if the carry is not set and these are literals. + __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); + __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); + + // Advance the input past the control block. + inputCursor += CONTROL_BLOCK_SIZE; + + { + // Pull out the bottom halves off the SSE registers from before - we want these + // things in GPRs for the more linear logic. + uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_HALF_LO( true, false ); + DECODE_STEP_HALF_HI( true, false ); + } + + { + // Now the top halves. + uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_END_LO( true, false ); + DECODE_STEP_END_HI( true, false ); + } + } + + // Decoding loop with no buffer checks, but will end at a safe distance from the end of the buffers. + // Note, when we get here we have already reached the point in the output buffer which is *past* where we can underflow + // due to a bad match offset. + while ( outputCursor < outputSafeEnd && inputCursor < inputSafeEnd ) + { + // This code is the same as the loop above, see comments there + __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); + __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); + __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); + + __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); + __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); + __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); + __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); + __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); // where we take the carry from the previous hi values + + previousCarryHi = carryHi; + + __m128i neg1 = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); + + __m128i bytesOutLo = _mm_sub_epi8( controlLo, _mm_xor_si128( shiftedCarryHi, neg1 ) ); + __m128i bytesOutHi = _mm_sub_epi8( controlHi, _mm_xor_si128( carryLo, neg1 ) ); + + __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_min_epi8( literalsPerControl, bytesOutLo ) ); + __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_min_epi8( literalsPerControl, bytesOutHi ) ); + + __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), neg1 ); + __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), neg1 ); + + __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); + __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); + + inputCursor += CONTROL_BLOCK_SIZE; + + { + uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_HALF_LO( false, false ); + DECODE_STEP_HALF_HI( false, false ); + } + + { + uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_END_LO( false, false ); + DECODE_STEP_END_HI( false, false ); + } + } + } + + // Decoding loop with all buffer checks. + { + const uint8_t* inputEarlyEnd; + uint8_t* outputEarlyEnd; + inputEarlyEnd = (( input + inputLength ) - END_PADDING_LITERALS); + outputEarlyEnd = ( output + outputLength ) - END_PADDING_LITERALS; + + while ( outputCursor < outputEarlyEnd && inputCursor < inputEarlyEnd ) + { + __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); + __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); + __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); + + __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); + __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); + __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); + __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); + __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); // where we take the carry from the previous hi values + + previousCarryHi = carryHi; + + __m128i neg1 = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); + + __m128i bytesOutLo = _mm_sub_epi8( controlLo, _mm_xor_si128( shiftedCarryHi, neg1 ) ); + __m128i bytesOutHi = _mm_sub_epi8( controlHi, _mm_xor_si128( carryLo, neg1 ) ); + + __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_min_epi8( literalsPerControl, bytesOutLo ) ); + __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_min_epi8( literalsPerControl, bytesOutHi ) ); + + __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), neg1 ); + __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), neg1 ); + + __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); + __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); + + inputCursor += CONTROL_BLOCK_SIZE; + + if ( inputCursor > inputEarlyEnd ) + goto BUFFER_END; + + { + uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_HALF_LO( true, true ); + DECODE_STEP_HALF_HI( true, true ); + } + + { + uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_END_LO( true, true ); + DECODE_STEP_END_HI( true, true ); + } + } + +BUFFER_END: + + // When we get here, we have either advanced the right amount on both cursors + // or something bad happened, so leave it as is, so we can tell where + // the error happened. + if ( inputCursor == inputEarlyEnd && outputCursor == outputEarlyEnd ) + { + size_t remainingLiterals = ( input + inputLength ) - inputCursor; + + // copy any trailing literals + memcpy( outputCursor, inputCursor, remainingLiterals ); + + outputCursor += remainingLiterals; + } + } + +MATCH_UNDERFLOW: + + return outputCursor - output; +} diff --git a/contrib/lzsse/lzsse2/lzsse2.h b/contrib/lzsse/lzsse2/lzsse2.h new file mode 100644 index 00000000000..ea908b85fb5 --- /dev/null +++ b/contrib/lzsse/lzsse2/lzsse2.h @@ -0,0 +1,90 @@ +/* +Copyright (c) 2016, Conor Stokes +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef LZSSE2_H__ +#define LZSSE2_H__ + +#pragma once + +/* LZSSE2 - x64/SSE targeted codec for better performance with high compression ratio data/more optimal compressors. + * Supports minimum 3 byte matches, maximum 16 bytes of match per control word and 2 byte literal runs per control word. + */ + +#ifdef __cplusplus +extern "C" +{ +#endif + + /* Re-usable parse state object for compression. */ +typedef struct LZSSE2_OptimalParseState LZSSE2_OptimalParseState; + +/* Allocate the parse state for compression - returns null on failure. Note + Buffersize has to be greater or equal to any inputLength used with LZSSE2_CompressOptimalParse */ +LZSSE2_OptimalParseState* LZSSE2_MakeOptimalParseState( size_t bufferSize ); + +/* De-allocate the parse state for compression */ +void LZSSE2_FreeOptimalParseState( LZSSE2_OptimalParseState* toFree ); + +/* "Optimal" compression routine. +* Will compress data into LZSSE2 format, uses hash BST matching to find matches and run an optimal parse (high relative memory usage). Requires SSE 4.1. +* state : Contains the hash table for matching, passed as a parameter so that allocations can be re-used. +* input : Buffer containing uncompressed data to be compressed. May not be null. +* inputLength : Length of the compressed data in the input buffer - note should be under 2GB. +* output : Buffer that will receive the compressed output. +* outputLength : The length reserved in the buffer for compressed data. This should be at least inputLength. Note, +* The compressed data should never be longer than inputLength, as in this case the data is stored raw. +* level : The compression level to use for this file 1->17, 17 is highest compression, 0 is least +* Thread Safety - state can not be used on multiple threads with calls running concurrently. Can run multiple threads with separate state +* concurrently. +* +* Returns the size of the compressed data, or 0 in the case of error (e.g. outputLength is less than inputLength). +*/ +size_t LZSSE2_CompressOptimalParse( LZSSE2_OptimalParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength, unsigned int level ); + +/* Decompression routine. +* This routine will decompress data in the LZSSE2 format and currently requires SSE 4.1 and is targeted at x64. +* It will perform poorly on x86 due to hunger for registers. +* input : Buffer containing compressed input block. May not be null. +* inputLength : Length of the compressed data in the input buffer - note, this should be under 2GB +* output : Buffer that will received the de-compressed output. Note, that this needs to be at least outputLength long. +* May not be null. +* outputLength : The length of the compressed output - note, this should be under 2GB +* +* Provided that input and output are valid pointers to buffers of at least their specified size, this routine +* should be memory safe - both match pointer checks and input/output buffer checks exist. +* +* Returns the size of the decompressed data, which will be less than outputLength in the event of an error (number of bytes +* will indicate where in the output stream the error occured). +* +* Note that this data is not hash verified, errors that occur are either from a misformed stream or bad buffer sizes. +* Remember, corrupt data can still be valid to decompress. +*/ +size_t LZSSE2_Decompress( const void* input, size_t inputLength, void* output, size_t outputLength ); + +#ifdef __cplusplus +} +#endif + +#endif /* -- LZSSE2_H__ */ diff --git a/contrib/lzsse/lzsse2/lzsse2_platform.h b/contrib/lzsse/lzsse2/lzsse2_platform.h new file mode 100644 index 00000000000..5cbab6b0600 --- /dev/null +++ b/contrib/lzsse/lzsse2/lzsse2_platform.h @@ -0,0 +1,73 @@ +/* +Copyright (c) 2016, Brian Marshall +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef LZSSE2_PLATFORM_H__ +#define LZSSE2_PLATFORM_H__ + +#pragma once + +/* + Compiler/Platform detection based on the table from: + https://blogs.msdn.microsoft.com/vcblog/2015/12/04/clang-with-microsoft-codegen-in-vs-2015-update-1/ +*/ + +#ifdef _MSC_VER + +/* + Microsoft Visual Studio Support. + C1xx/C2, Clang/C2 and Clang/LLVM all support the Microsoft header files and _BitScanForward + + Note: if you receive errors with the intrinsics make sure that you have SSE4.1 support enabled. + For example with Clang include "-msse4.1" on the command line +*/ +#include + +#else /* _MSC_VER */ + +#ifdef __GNUC__ + +/* + GCC +*/ + +/* + Note: including just would be sufficient, but including x86intrin is a better match to intrin.h on Visual Studio as + both include all intrinsics for the enabled processor, rather than just SSE4.1. +*/ +#include +/* _BitScanForward is Visual Studio specific. */ +#define _BitScanForward(x, m) *(x) = __builtin_ctz(m) + +#else + +/* +If you hit the error below, then add detection for your compiler/platform to this header file. +*/ +#error Platform not supported + +#endif /* __GNUC__ */ +#endif /* _MSC_VER */ + +#endif /* -- LZSSE2_PLATFORM_H__ */ diff --git a/contrib/lzsse/lzsse4/lzsse4.cpp b/contrib/lzsse/lzsse4/lzsse4.cpp new file mode 100644 index 00000000000..cf9f6fc6e60 --- /dev/null +++ b/contrib/lzsse/lzsse4/lzsse4.cpp @@ -0,0 +1,1499 @@ +/* +Copyright (c) 2016, Conor Stokes +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include +#include +#include +#include + +#include "lzsse4_platform.h" +#include "lzsse4.h" + +#pragma warning ( disable : 4127 ) + +namespace +{ + // Constants - most of these should not be changed without corresponding code changes because it will break many things in unpredictable ways. + const uint32_t WINDOW_BITS = 16; + const uint32_t MIN_MATCH_LENGTH = 4; + const uint32_t LZ_WINDOW_SIZE = 1 << WINDOW_BITS; + const uint32_t LZ_WINDOW_MASK = LZ_WINDOW_SIZE - 1; + const uint32_t FAST_HASH_BITS = 20; // You can change this - more bits = more matches, less bits = more cache hits + const uint32_t FAST_BUCKETS_COUNT = 1 << FAST_HASH_BITS; + const uint32_t FAST_HASH_MASK = FAST_BUCKETS_COUNT - 1; + const uint32_t MIN_COMPRESSION_SIZE = 32; + const uint32_t END_PADDING_LITERALS = 16; + const int32_t NO_MATCH = -1; + const int32_t EMPTY_NODE = -1; + const uint32_t MIN_LITERAL_COUNT = 8; + const uint32_t CONTROL_BITS = 4; + const uint32_t LITERAL_BITS = 8; + const uint32_t OFFSET_BITS = 16; + const uint32_t BASE_MATCH_BITS = OFFSET_BITS + CONTROL_BITS; + const uint32_t OFFSET_SIZE = 2; + const uint32_t EXTENDED_MATCH_BOUND = ( 1 << CONTROL_BITS ) - 1; + const uint32_t CONTROL_BLOCK_SIZE = sizeof( __m128i ); + const uint32_t CONTROLS_PER_BLOCK = 32; + const uint32_t LITERALS_PER_CONTROL = 4; + const uint32_t MAX_INPUT_PER_CONTROL = 4; + const size_t OUTPUT_BUFFER_SAFE = EXTENDED_MATCH_BOUND * CONTROLS_PER_BLOCK; + const size_t INPUT_BUFFER_SAFE = MAX_INPUT_PER_CONTROL * CONTROLS_PER_BLOCK; + const uint16_t INITIAL_OFFSET = MIN_MATCH_LENGTH; + const uint32_t OPTIMAL_HASH_BITS = 20; + const uint32_t OPTIMAL_BUCKETS_COUNT = 1 << OPTIMAL_HASH_BITS; + const uint32_t OPTIMAL_HASH_MASK = OPTIMAL_BUCKETS_COUNT - 1; + const size_t SKIP_MATCH_LENGTH = 128; + const uint32_t NO_SKIP_LEVEL = 17; +} + + +struct LZSSE4_FastParseState +{ + int32_t buckets[ FAST_BUCKETS_COUNT ]; // stores the first matching position, we can then look at the rest of the matches by tracing through the window. +}; + + +LZSSE4_FastParseState* LZSSE4_MakeFastParseState() +{ + return new LZSSE4_FastParseState(); +} + + +void LZSSE4_FreeFastParseState( LZSSE4_FastParseState* toFree ) +{ + delete toFree; +} + + +inline void SetHash( LZSSE4_FastParseState* state, uint32_t hash, const uint8_t* input, const uint8_t* inputCursor ) +{ + int32_t position = static_cast( inputCursor - input ); + + state->buckets[ hash & FAST_HASH_MASK ] = position; +} + +// Simple fast hash function - actually what is used in snappy and derivatives +// There's probably better, but I haven't spent time focusing on this area yet. +inline uint32_t HashFast( const uint8_t* inputCursor ) +{ + return *reinterpret_cast( inputCursor ) * 0x1e35a7bd >> ( 32 - FAST_HASH_BITS ); +} + +size_t LZSSE4_CompressFast( LZSSE4_FastParseState* state, const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength ) +{ + if ( outputLength < inputLength ) + { + // error case, output buffer not large enough. + return 0; + } + + const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); + uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); + + if ( inputLength < MIN_COMPRESSION_SIZE ) + { + memcpy( output, input, inputLength ); + + return inputLength; + } + + const uint8_t* inputCursor = input; + const uint8_t* inputEnd = input + inputLength; + const uint8_t* inputEarlyEnd = inputEnd - END_PADDING_LITERALS; + uint8_t* outputCursor = output; + uint8_t* outputEarlyEnd = ( output + outputLength ) - END_PADDING_LITERALS; + uint32_t hash = 0; + + // initialize hash to empty + for ( int32_t* where = state->buckets, *end = state->buckets + FAST_BUCKETS_COUNT; where < end; where += 4 ) + { + where[ 0 ] = -1; + where[ 1 ] = -1; + where[ 2 ] = -1; + where[ 3 ] = -1; + } + + // initial literals that wont be compressed + for ( uint32_t where = 0; where < MIN_MATCH_LENGTH; ++where ) + { + hash = HashFast( inputCursor ); + + SetHash( state, hash, input, inputCursor ); + + *( outputCursor++ ) = *( inputCursor++ ); + } + + uint8_t* currentControlBlock = outputCursor; + uint32_t currentControlCount = 0; + uint16_t previousOffset = INITIAL_OFFSET; + size_t literalsToFlush = 0; + + outputCursor += CONTROL_BLOCK_SIZE; + + bool lastControlIsNop = false; + + // Loop through the data until we hit the end of one of the buffers (minus the end padding literals) + while ( inputCursor < inputEarlyEnd && outputCursor <= outputEarlyEnd ) + { + lastControlIsNop = false; + + hash = HashFast( inputCursor ); + + int matchPosition = state->buckets[ hash & FAST_HASH_MASK ]; + int currentPosition = static_cast< int32_t >( inputCursor - input ); + uint32_t matchLength = 0; + uint16_t matchOffset = static_cast< uint16_t >( currentPosition - matchPosition ); + + // If we had a hit in the hash and it wasn't outside the window. + if ( matchPosition >= 0 && ( currentPosition - matchPosition ) < ( LZ_WINDOW_SIZE - 1 ) ) + { + const uint8_t* matchCandidate = input + matchPosition; + uint32_t lengthToEnd = static_cast< uint32_t >( inputEarlyEnd - inputCursor ); + // Here we limit the hash length to prevent overlap matches with offset less than 16 bytes + uint32_t maxLength = matchOffset <= ( EXTENDED_MATCH_BOUND + 1 ) && matchOffset < lengthToEnd ? matchOffset : lengthToEnd; + + // Find how long the match is 16 bytes at a time. + while ( matchLength < maxLength ) + { + __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); + __m128i match16 = _mm_loadu_si128( reinterpret_cast( matchCandidate + matchLength ) ); + + unsigned long matchBytes; + + // Finds the number of equal bytes at the start of the 16 + _BitScanForward( &matchBytes, ( static_cast< unsigned long >( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); + + matchLength += matchBytes; + + if ( matchBytes != sizeof( __m128i ) ) + { + break; + } + } + + matchLength = matchLength < maxLength ? matchLength : maxLength; + } + + // If we have at least the minimum match length (4 bytes) + if ( matchLength >= MIN_MATCH_LENGTH ) + { + // Do we have literals to flush before the match? + if ( literalsToFlush > 0 ) + { + // Start a new control block if we need one. + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + + // Would be larger than compressed size, get out! + if ( outputCursor > outputEarlyEnd ) + { + break; + } + } + + currentControlBlock[ currentControlCount >> 1 ] = + ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( literalsToFlush - 1 ) << 4 ); + + // flush the literals. + // note the xor against the data that would be read in the match. + for ( uint32_t where = 0; where < literalsToFlush; ++where ) + { + const uint8_t* currentInput = inputCursor - ( literalsToFlush - where ); + + *( outputCursor++ ) = *currentInput ^ *( currentInput - previousOffset ); + } + + ++currentControlCount; + + literalsToFlush = 0; + + // Would be larger than compressed size, get out! + if ( outputCursor > outputEarlyEnd ) + { + break; + } + } + + // Start a new control block if the previous one is full. + // Note this is done before the offset is written out - the offset + // is always written after the control block containing the first + // control in the match. + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + + if ( outputCursor > outputEarlyEnd ) + { + break; + } + } + + // The match length value we are encoding. + size_t toEncode = matchLength; + + // Write the offset out - note the xor with the previous offset. + *reinterpret_cast< uint16_t* >( outputCursor ) = matchOffset ^ previousOffset; + + previousOffset = matchOffset; + outputCursor += sizeof( uint16_t ); + + for ( ;; ) + { + // Check if we need to start a new control block + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + + if ( outputCursor > outputEarlyEnd ) + { + break; + } + } + + // If the encode size is greater than we can hold in a control, write out a full match length + // control, subtract full control value from the amount to encode and loop around again. + if ( toEncode >= EXTENDED_MATCH_BOUND ) + { + currentControlBlock[ currentControlCount >> 1 ] = + ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( EXTENDED_MATCH_BOUND ) << 4 ); + + toEncode -= EXTENDED_MATCH_BOUND; + + ++currentControlCount; + } + else // Write out the remaining match length control. Could potentially be zero. + { + currentControlBlock[ currentControlCount >> 1 ] = + ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( toEncode ) << 4 ); + + if ( currentControlCount == 0 && toEncode == 0 ) + { + lastControlIsNop = true; + } + + ++currentControlCount; + + break; + } + } + + // Update the value into the hash for future matches. + SetHash( state, hash, input, inputCursor ); + + ++inputCursor; + + // Hash all the other values in the match too. + for ( const uint8_t* nextArrival = inputCursor + matchLength - 1; inputCursor < nextArrival; ++inputCursor ) + { + hash = HashFast( inputCursor ); + SetHash( state, hash, input, inputCursor ); + } + } + else + { + // One more literal to write out. + ++literalsToFlush; + + // If we have reached the maximum number of literals allowed in the control, flush them out. + if ( literalsToFlush == LITERALS_PER_CONTROL ) + { + // Check if the control block is full and we need start a new one. + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + + if ( outputCursor > outputEarlyEnd ) + { + break; + } + } + + currentControlBlock[ currentControlCount >> 1 ] = + ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( ( static_cast( LITERALS_PER_CONTROL - 1 ) ) << 4 ); + + ++currentControlCount; + + *reinterpret_cast< uint32_t* >( outputCursor ) = + *reinterpret_cast< const uint32_t* >( inputCursor - 3 ) ^ + *reinterpret_cast< const uint32_t* >( ( inputCursor - 3 ) - previousOffset ); + + outputCursor += 4; + + //*( outputCursor++ ) = *( inputCursor - 3 ) ^ *( ( inputCursor - 3 ) - previousOffset ); + //*( outputCursor++ ) = *( inputCursor - 2 ) ^ *( ( inputCursor - 2 ) - previousOffset ); + //*( outputCursor++ ) = *( inputCursor - 1 ) ^ *( ( inputCursor - 1 ) - previousOffset ); + //*( outputCursor++ ) = *inputCursor ^ *( inputCursor - previousOffset ); + + if ( outputCursor > outputEarlyEnd ) + { + break; + } + + literalsToFlush = 0; + } + + // Update the hash with this byte + SetHash( state, hash, input, inputCursor ); + + ++inputCursor; + } + } + + // If we would create a compression output bigger than or equal to the input, just copy the input to the output and return equal size. + if ( ( ( outputCursor + literalsToFlush + ( currentControlCount == CONTROLS_PER_BLOCK ? CONTROL_BLOCK_SIZE : 0 ) ) ) >= output + inputLength - END_PADDING_LITERALS ) + { + memcpy( output, input, inputLength ); + + outputCursor = output + inputLength; + } + else + { + // Flush any remaining literals. + if ( literalsToFlush > 0 ) + { + lastControlIsNop = false; + + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + } + + currentControlBlock[ currentControlCount >> 1 ] = + ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( literalsToFlush - 1 ) << 4 ); + + for ( uint32_t where = 0; where < literalsToFlush; ++where ) + { + const uint8_t* currentInput = inputCursor - ( literalsToFlush - where ); + + *( outputCursor++ ) = *currentInput ^ *( currentInput - previousOffset ); + } + + ++currentControlCount; + } + + // Need to finish off shifting the final control block into the low nibble if there is no second nibble + if ( ( currentControlCount & 1 ) > 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] >>= 4; + } + + if ( lastControlIsNop ) + { + outputCursor -= CONTROL_BLOCK_SIZE; + } + + size_t remainingLiterals = ( input + inputLength ) - inputCursor; + + // copy remaining literals + memcpy( outputCursor, inputCursor, remainingLiterals ); + + outputCursor += remainingLiterals; + } + + // Return the size of the compressed data. + return outputCursor - output; +} + + +struct Arrival +{ + size_t cost; + int32_t from; + int32_t to; + uint16_t offset; +}; + +struct TreeNode +{ + int32_t children[ 2 ]; +}; + +struct LZSSE4_OptimalParseState +{ + // Note, we should really replace this with a BST, hash chaining works but is *slooooooooooooooow* for optimal parse. + int32_t roots[ OPTIMAL_BUCKETS_COUNT ]; + + TreeNode window[ LZ_WINDOW_SIZE ]; + + Arrival* arrivals; + + size_t bufferSize; +}; + + +LZSSE4_OptimalParseState* LZSSE4_MakeOptimalParseState( size_t bufferSize ) +{ + if ( bufferSize > 0 && ( SIZE_MAX / sizeof( Arrival ) ) < bufferSize ) + { + return nullptr; + } + + LZSSE4_OptimalParseState* result = reinterpret_cast< LZSSE4_OptimalParseState* >( ::malloc( sizeof( LZSSE4_OptimalParseState ) ) ); + + result->bufferSize = bufferSize; + + if ( result != nullptr ) + { + result->arrivals = reinterpret_cast< Arrival* >( ::malloc( sizeof( Arrival ) * bufferSize ) ); + + if ( result->arrivals == nullptr ) + { + LZSSE4_FreeOptimalParseState( result ); + + result = nullptr; + } + } + + return result; +} + + +void LZSSE4_FreeOptimalParseState( LZSSE4_OptimalParseState* toFree ) +{ + ::free( toFree->arrivals ); + + toFree->arrivals = nullptr; + + ::free( toFree ); +} + + +inline uint32_t HashOptimal( const uint8_t* inputCursor ) +{ + return *reinterpret_cast( inputCursor ) * 0x1e35a7bd >> ( 32 - OPTIMAL_HASH_BITS ); +} + + +struct Match +{ + size_t length; + int32_t position; + uint16_t offset; +}; + + +inline Match SearchAndUpdateFinder( LZSSE4_OptimalParseState& state, const uint8_t* input, const uint8_t* inputCursor, const uint8_t* inputEnd, uint32_t cutOff ) +{ + Match result; + + int32_t position = static_cast( inputCursor - input ); + + result.position = NO_MATCH; + result.length = MIN_MATCH_LENGTH; + result.offset = 0; + + size_t lengthToEnd = inputEnd - inputCursor; + int32_t lastPosition = position - ( LZ_WINDOW_SIZE - 1 ); + uint32_t hash = HashOptimal( inputCursor ); + + lastPosition = lastPosition > 0 ? lastPosition : 0; + + int32_t treeCursor = state.roots[ hash ]; + + state.roots[ hash ] = position; + + int32_t* left = &state.window[ position & LZ_WINDOW_MASK ].children[ 1 ]; + int32_t* right = &state.window[ position & LZ_WINDOW_MASK ].children[ 0 ]; + size_t leftLength = 0; + size_t rightLength = 0; + + for ( ;; ) + { + if ( cutOff-- == 0 || treeCursor < lastPosition ) + { + *left = *right = EMPTY_NODE; + break; + } + + TreeNode& currentNode = state.window[ treeCursor & LZ_WINDOW_MASK ]; + const uint8_t* key = input + treeCursor; + size_t matchLength = leftLength < rightLength ? leftLength : rightLength; + + uint16_t matchOffset = static_cast< uint16_t >( position - treeCursor ); + size_t maxLength = matchOffset <= ( EXTENDED_MATCH_BOUND + 1 ) && matchOffset < lengthToEnd ? matchOffset : lengthToEnd; + + while ( matchLength < lengthToEnd ) + { + __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); + __m128i match16 = _mm_loadu_si128( reinterpret_cast( key + matchLength ) ); + + unsigned long matchBytes; + + _BitScanForward( &matchBytes, ( static_cast( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); + + matchLength += matchBytes; + + if ( matchBytes != 16 ) + { + break; + } + } + + matchLength = matchLength < lengthToEnd ? matchLength : lengthToEnd; + + size_t truncatedMatchLength = matchLength < maxLength ? matchLength : maxLength; + + if ( truncatedMatchLength >= result.length && matchOffset >= LITERALS_PER_CONTROL ) + { + result.length = truncatedMatchLength; + result.offset = matchOffset; + result.position = treeCursor; + } + + if ( matchLength == lengthToEnd ) + { + *left = currentNode.children[ 1 ]; + *right = currentNode.children[ 0 ]; + break; + } + + if ( inputCursor[ matchLength ] < key[ matchLength ] || ( matchLength == lengthToEnd ) ) + { + *left = treeCursor; + left = currentNode.children; + treeCursor = *left; + leftLength = matchLength; + } + else + { + *right = treeCursor; + right = currentNode.children + 1; + treeCursor = *right; + rightLength = matchLength; + } + } + + // Special RLE overlapping match case, the LzFind style match above doesn't work very well with our + // restriction of overlapping matches having offsets of at least 16. + // Suffix array seems like a better option to handling this. + { + // Note, we're detecting long RLE here, but if we have an offset too close, we'll sacrifice a fair + // amount of decompression performance to load-hit-stores. + int32_t matchPosition = position - ( sizeof( __m128i ) * 2 ); + + if ( matchPosition >= 0 ) + { + uint16_t matchOffset = static_cast( position - matchPosition ); + const uint8_t* key = input + matchPosition; + size_t matchLength = 0; + + while ( matchLength < lengthToEnd ) + { + __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); + __m128i match16 = _mm_loadu_si128( reinterpret_cast( key + matchLength ) ); + + unsigned long matchBytes; + + _BitScanForward( &matchBytes, ( static_cast( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); + + matchLength += matchBytes; + + if ( matchBytes != 16 ) + { + break; + } + + } + + matchLength = matchLength < lengthToEnd ? matchLength : lengthToEnd; + + if ( matchLength >= result.length ) + { + result.length = matchLength; + result.offset = matchOffset; + result.position = matchPosition; + } + } + } + + return result; +} + + +size_t LZSSE4_CompressOptimalParse( LZSSE4_OptimalParseState* state, const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength, unsigned int level ) +{ + if ( outputLength < inputLength || state->bufferSize < inputLength ) + { + // error case, output buffer not large enough. + return 0; + } + + const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); + uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); + + if ( inputLength < MIN_COMPRESSION_SIZE ) + { + memcpy( output, input, inputLength ); + + return inputLength; + } + + const uint8_t* inputCursor = input; + const uint8_t* inputEnd = input + inputLength; + Arrival* arrivalWatermark = state->arrivals; + Arrival* arrival = state->arrivals; + uint32_t cutOff = 1 << level; + + for ( int32_t* rootCursor = state->roots, *end = rootCursor + OPTIMAL_BUCKETS_COUNT; rootCursor < end; rootCursor += 4 ) + { + rootCursor[ 0 ] = EMPTY_NODE; + rootCursor[ 1 ] = EMPTY_NODE; + rootCursor[ 2 ] = EMPTY_NODE; + rootCursor[ 3 ] = EMPTY_NODE; + } + + for ( uint32_t where = 0; where < MIN_MATCH_LENGTH; ++where ) + { + SearchAndUpdateFinder( *state, input, inputCursor, inputEnd - END_PADDING_LITERALS, cutOff ); + + ++inputCursor; + } + + arrival->cost = LITERAL_BITS * LITERALS_PER_CONTROL; + arrival->from = -1; + arrival->offset = 0; + + // loop through each character and project forward the matches at that character to calculate the cheapest + // path of arrival for each individual character. + for ( const uint8_t* earlyEnd = inputEnd - END_PADDING_LITERALS; inputCursor < earlyEnd; ++inputCursor, ++arrival ) + { + uint32_t lengthToEnd = static_cast< uint32_t >( earlyEnd - inputCursor ); + int32_t currentPosition = static_cast< int32_t >( inputCursor - input ); + size_t literalsForward = LITERALS_PER_CONTROL < lengthToEnd ? LITERALS_PER_CONTROL : lengthToEnd; + size_t arrivalCost = arrival->cost; + + // NOTE - we currently assume only 2 literals filled in here, because the minimum match length is 3. + // If we wanted to go with a higher minimum match length, we would need to fill in more literals before hand. + // Also, because there is a maximum of 2 literals per control block assumed. + + // project forward the cost of a single literal + + for ( size_t where = 1; where <= literalsForward; ++where ) + { + Arrival* literalArrival = arrival + where; + size_t literalCost = arrivalCost + CONTROL_BITS + ( where * LITERAL_BITS ); + + if ( literalArrival > arrivalWatermark || literalArrival->cost > literalCost ) + { + literalArrival->cost = literalCost; + literalArrival->from = currentPosition; + literalArrival->offset = 0; + + arrivalWatermark = literalArrival > arrivalWatermark ? literalArrival : arrivalWatermark; + } + } + + Match match = SearchAndUpdateFinder( *state, input, inputCursor, earlyEnd, cutOff ); + + if ( match.position != NO_MATCH ) + { + for ( size_t matchedLength = MIN_MATCH_LENGTH, end = match.length + 1; matchedLength < end; ++matchedLength ) + { + Arrival* matchArrival = arrival + matchedLength; + size_t matchCost = arrivalCost + BASE_MATCH_BITS; + + if ( matchedLength >= EXTENDED_MATCH_BOUND ) + { + matchCost += ( matchedLength / EXTENDED_MATCH_BOUND ) * CONTROL_BITS; + } + + if ( matchArrival > arrivalWatermark || matchArrival->cost > matchCost ) + { + matchArrival->cost = matchCost; + matchArrival->from = currentPosition; + matchArrival->offset = match.offset; + + arrivalWatermark = matchArrival > arrivalWatermark ? matchArrival : arrivalWatermark; + } + } + + if ( match.length > SKIP_MATCH_LENGTH && level < NO_SKIP_LEVEL ) + { + arrival += match.length - LITERALS_PER_CONTROL; + inputCursor += match.length - LITERALS_PER_CONTROL; + } + } + } + + // If this would cost more to encode than it would if it were just literals, encode it with no control blocks, + // just literals + if ( ( arrivalWatermark->cost + END_PADDING_LITERALS * LITERAL_BITS + CONTROLS_PER_BLOCK * CONTROL_BITS ) > ( inputLength * LITERAL_BITS ) ) + { + memcpy( output, input, inputLength ); + + return inputLength; + } + + Arrival* previousPathNode; + + // now trace the actual optimal parse path back, connecting the nodes in the other direction. + for ( const Arrival* pathNode = arrivalWatermark; pathNode->from > 0; pathNode = previousPathNode ) + { + previousPathNode = state->arrivals + ( pathNode->from - LITERALS_PER_CONTROL ); + + previousPathNode->to = static_cast( ( pathNode - state->arrivals ) + LITERALS_PER_CONTROL ); + } + + uint8_t* outputCursor = output; + + memcpy( outputCursor, input, MIN_MATCH_LENGTH ); + + outputCursor += MIN_MATCH_LENGTH; + + uint8_t* currentControlBlock = outputCursor; + uint32_t currentControlCount = 0; + uint32_t totalControlCount = 0; + + outputCursor += CONTROL_BLOCK_SIZE; + + Arrival* nextPathNode; + + size_t totalPathLength = MIN_MATCH_LENGTH; + uint16_t previousOffset = INITIAL_OFFSET; + + bool lastControlIsNop = false; + + // Now walk forwards again and actually write out the data. + for ( const Arrival* pathNode = state->arrivals; pathNode < arrivalWatermark; pathNode = nextPathNode ) + { + int32_t currentPosition = static_cast< int32_t >( ( pathNode - state->arrivals ) + LITERALS_PER_CONTROL ); + + nextPathNode = state->arrivals + ( pathNode->to - LITERALS_PER_CONTROL ); + + size_t pathDistance = nextPathNode - pathNode; + + totalPathLength += pathDistance; + + lastControlIsNop = false; + + if ( nextPathNode->offset == 0 ) + { + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + currentControlCount = 0; + } + + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + ( static_cast( pathDistance ) - 1 ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + ( static_cast< uint8_t >( pathDistance ) - 1 ) << CONTROL_BITS; + } + + // output the literals. + for ( int32_t where = 0; where < pathDistance; ++where ) + { + const uint8_t* currentInput = input + currentPosition + where; + + outputCursor[ where ] = *currentInput ^ *( currentInput - previousOffset ); + } + + outputCursor += pathDistance; + + ++totalControlCount; + ++currentControlCount; + } + else + { + // Check if we need to start a new control block + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + } + + // Write the offset out - note the xor with the previous offset. + *reinterpret_cast< uint16_t* >( outputCursor ) = nextPathNode->offset ^ previousOffset; + + previousOffset = nextPathNode->offset; + outputCursor += sizeof( uint16_t ); + + if ( pathDistance < EXTENDED_MATCH_BOUND ) + { + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + static_cast( pathDistance ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + static_cast< uint8_t >( pathDistance ) << CONTROL_BITS; + } + + ++currentControlCount; + } + else + { + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + static_cast( EXTENDED_MATCH_BOUND ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + static_cast< uint8_t >( EXTENDED_MATCH_BOUND ) << CONTROL_BITS; + } + + ++currentControlCount; + + size_t toEncode = pathDistance - EXTENDED_MATCH_BOUND; + + for ( ;; ) + { + // Check if we need to start a new control block + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + } + + // If the encode size is greater than we can hold in a control, write out a full match length + // control, subtract full control value from the amount to encode and loop around again. + if ( toEncode >= EXTENDED_MATCH_BOUND ) + { + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + static_cast( EXTENDED_MATCH_BOUND ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + static_cast< uint8_t >( EXTENDED_MATCH_BOUND ) << CONTROL_BITS; + } + + toEncode -= EXTENDED_MATCH_BOUND; + + ++currentControlCount; + } + else // Write out the remaining match length control. Could potentially be zero. + { + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + static_cast( toEncode ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + static_cast< uint8_t >( toEncode ) << CONTROL_BITS; + } + + if ( toEncode == 0 && currentControlCount == 0 ) + { + lastControlIsNop = true; + } + + ++currentControlCount; + + break; + } + } + } + } + } + + if ( lastControlIsNop ) + { + outputCursor -= CONTROL_BLOCK_SIZE; + } + + size_t remainingLiterals = ( input + inputLength ) - inputCursor; + + // copy remaining literals + memcpy( outputCursor, inputCursor, remainingLiterals ); + + outputCursor += remainingLiterals; + + return outputCursor - output; +} + + +size_t LZSSE4_Decompress( const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength ) +{ + const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); + uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); + + // Data was not compressible, just copy initial values + if ( outputLength == inputLength ) + { + memcpy( output, input, outputLength ); + + return inputLength; + } + + const uint8_t* inputCursor = input; + uint8_t* outputCursor = output; + + // The offset starts off as the minimum match length. We actually need it least four + // characters back because we need them to be set to xor out the literals from the match data. + size_t offset = INITIAL_OFFSET; + __m128i previousCarryHi = _mm_setzero_si128(); + + // Copy the initial literals to the output. + for ( uint32_t where = 0; where < MIN_MATCH_LENGTH; ++where ) + { + *( outputCursor++ ) = *( inputCursor++ ); + } + + // Let me be clear, I am usually anti-macro, but they work for this particular (very unusual) case. + // DECODE_STEP is a regular decoding step, DECODE_STEP_HALF and DECODE_STEP_END are because the compiler couldn't + // seem to remove some of the dead code where values were updated and then never used. + + // What these macros do: + // Decode a single literal run or match run for a single control nibble. + // How they do it: + // - Read the *unaligned* input (in the case of LZSSE-F - twice), it goes into both a regular variable and an SSE register, + // because it could either be literals or an offset (or nothing at all). The low byte of streamBytesRead controls how much we advance + // the input cursor. + // - Used a contived set of casts to sign extend the "read offset" control mask and then use it to mask the input word, + // which is then xor'd against the offset, for a "branchless" conditional move into the offset which + // has been carried over from the previous literal/match block. Note, this ends up doing better than a cmov on most + // modern processors. But we need to pre-xor the input offset. + // - We then load the match data from output buffer (offset back from the current output point). Unconditional load here. + // - We broadcast the "from literal" control mask from the current least significant byte of the SSE register using a shuffle epi-8 + // - We mask the literals with that SSE register wide mask. + // - The literals have been pre-xor'd with the data read in as match data, so we use an xor to branchlessly choose between the two. + // In this case, it ends up a better option than a blendv on most processors. + // - Store the block. We store all 16 bytes of the SSE register (due to some constraints in the format of the data, we won't + // go past the end of the buffer), but we may overlap this. + // - bytesOut controls how much we advance the output cursor. + // - We use 8 bit shifts to advance all the controls up to the next byte. There is some variable sized register trickery that + // x86/x64 is great for as long as we don't anger the register renamer. + +#define DECODE_STEP( HILO, CHECKMATCH, CHECKBUFFERS ) \ + { \ + size_t inputWord = *reinterpret_cast( inputCursor ); \ + __m128i literals = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); \ + \ + offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ + \ + readOffsetHalf##HILO >>= 8; \ + \ + const uint8_t* matchPointer = reinterpret_cast( outputCursor - offset ); \ + \ + if ( CHECKMATCH && matchPointer < output ) \ + goto MATCH_UNDERFLOW; \ + \ + __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ + __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ + \ + literals = _mm_and_si128( literals, fromLiteral ); \ + \ + fromLiteral##HILO = _mm_srli_si128( fromLiteral##HILO, 1 ); \ + \ + __m128i toStore = _mm_xor_si128( matchData, literals ); \ + \ + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ + \ + outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ + inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ + \ + bytesOutHalf##HILO >>= 8; \ + streamBytesReadHalf##HILO >>= 8; \ + \ + if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ + goto BUFFER_END; \ + } + +#define DECODE_STEP_HALF( HILO, CHECKMATCH, CHECKBUFFERS ) \ + { \ + size_t inputWord = *reinterpret_cast( inputCursor ); \ + __m128i literals = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); \ + \ + offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ + \ + const uint8_t* matchPointer = reinterpret_cast( outputCursor - offset ); \ + \ + if ( CHECKMATCH && matchPointer < output ) \ + goto MATCH_UNDERFLOW; \ + \ + __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ + __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ + \ + literals = _mm_and_si128( literals, fromLiteral ); \ + \ + fromLiteral##HILO = _mm_srli_si128( fromLiteral##HILO, 1 ); \ + \ + __m128i toStore = _mm_xor_si128( matchData, literals ); \ + \ + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ + \ + outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ + inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ + \ + if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ + goto BUFFER_END; \ + } + +#define DECODE_STEP_END( HILO, CHECKMATCH, CHECKBUFFERS ) \ + { \ + size_t inputWord = *reinterpret_cast( inputCursor ); \ + __m128i literals = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); \ + \ + offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ + \ + const uint8_t* matchPointer = reinterpret_cast( outputCursor - offset ); \ + \ + if ( CHECKMATCH && matchPointer < output ) \ + goto MATCH_UNDERFLOW; \ + \ + __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ + __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ + \ + literals = _mm_and_si128( literals, fromLiteral ); \ + \ + __m128i toStore = _mm_xor_si128( matchData, literals ); \ + \ + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ + \ + outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ + inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ + \ + if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ + goto BUFFER_END; \ + } + +#define DECODE_STEP_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP( Lo, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP( Hi, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_HALF_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_HALF( Lo, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_HALF_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_HALF( Hi, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_END_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_END( Lo, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_END_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_END( Hi, CHECKMATCH, CHECKBUFFERS ) + + __m128i nibbleMask = _mm_set1_epi8( 0xF ); + __m128i offsetSize = _mm_set1_epi8( OFFSET_SIZE ); + + // Note, we use this block here because it allows the "fake" inputEarlyEnd/outputEarlyEnd not to cause register spills + // in the decompression loops. And yes, that did actually happen. + { + +#pragma warning ( push ) +#pragma warning ( disable : 4101 ) + + // These variables are not actually ever used in this block, because we use + // a constant conditional expression to take out the branches that would hit them. + // But unfortunately, we need them to compile. + const uint8_t* inputEarlyEnd; + uint8_t* outputEarlyEnd; + +#pragma warning ( pop ) + + // "Safe" ends to the buffer, before the input/output cursors hit these, we can loop without overflow checks. + const uint8_t* inputSafeEnd = ( input + inputLength ) - INPUT_BUFFER_SAFE; + uint8_t* outputSafeEnd = ( output + outputLength ) - OUTPUT_BUFFER_SAFE; + + // Decoding loop with offset output buffer underflow test, but no buffer overflow tests, assumed to end at a safe distance + // from overflows + while ( ( outputCursor - output ) < LZ_WINDOW_SIZE && outputCursor < outputSafeEnd && inputCursor < inputSafeEnd ) + { + // load the control block + __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); + + // split the control block into high and low nibbles + __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); + __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); + + // Note, the carries are set when the nibble is at its highest value, 15, meaning the operation after will + // be an extension of the current match operation. + + // Work out the carry for the low nibbles (which will be used with the high controls to put them into + // match without offset read mode). + __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); + + // The carry for the high nibbles is used with the low controls, but needs one byte from the previous iteration. We save + // the calculated carry to use that byte next iteration. + __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); + __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); + + previousCarryHi = carryHi; + + // We make the implicit assumption that the maximum number of literals to controls here is twice the offset size (4 vs 2), + // we are doing this here to save keeping the value around (spilling or fetching it each time) + __m128i literalsPerControl = _mm_add_epi8( offsetSize, offsetSize ); + + // Here we are testing if the runs will be literals or matches. Note that if the carries are set from the previous operation + // this will essentially be ignored later on. + __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); + __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); + + // Here we're calculating the number of bytes that will be output, we are actually subtracting negative one from the control + // (handy trick where comparison result masks are negative one) if carry is not set and it is a literal. + __m128i bytesOutLo = _mm_sub_epi8( controlLo, _mm_andnot_si128( shiftedCarryHi, isLiteralLo ) ); + __m128i bytesOutHi = _mm_sub_epi8( controlHi, _mm_andnot_si128( carryLo, isLiteralHi ) ); + + // Calculate the number of bytes to read per control. + // In the case the carry is set, no bytes. Otherwise, the offset size (2 bytes) for matches or the number of output bytes for literals. + __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_blendv_epi8( offsetSize, bytesOutLo, isLiteralLo ) ); + __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_blendv_epi8( offsetSize, bytesOutHi, isLiteralHi ) ); + + // I want 128 set bits please. + __m128i allSet = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); + + // Masks to read the offset (or keep the previous one) - set in the case that this is not a literal and the carry is not set + __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), allSet ); + __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), allSet ); + + // Masks whether we are reading literals - set if the carry is not set and these are literals. + __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); + __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); + + // Advance the input past the control block + inputCursor += CONTROL_BLOCK_SIZE; + + { + // Pull out the bottom halves off the SSE registers from before - we want these + // things in GPRs for the more linear logic. + uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_HALF_LO( true, false ); + DECODE_STEP_HALF_HI( true, false ); + } + + { + // Now the top halves. + uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_END_LO( true, false ); + DECODE_STEP_END_HI( true, false ); + } + } + + // Decoding loop with no buffer checks, but will end at a safe distance from the end of the buffers. + // Note, when we get here we have already reached the point in the output buffer which is *past* where we can underflow + // due to a bad match offset. + while ( outputCursor < outputSafeEnd && inputCursor < inputSafeEnd ) + { + // This code is the same as the loop above, see comments there + __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); + __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); + __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); + + __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); + __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); + __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); // where we take the carry from the previous hi values + + previousCarryHi = carryHi; + + __m128i literalsPerControl = _mm_add_epi8( offsetSize, offsetSize ); + __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); + __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); + + __m128i bytesOutLo = _mm_sub_epi8( controlLo, _mm_andnot_si128( shiftedCarryHi, isLiteralLo ) ); + __m128i bytesOutHi = _mm_sub_epi8( controlHi, _mm_andnot_si128( carryLo, isLiteralHi ) ); + + __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_blendv_epi8( offsetSize, bytesOutLo, isLiteralLo ) ); + __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_blendv_epi8( offsetSize, bytesOutHi, isLiteralHi ) ); + + __m128i neg1 = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); + + __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), neg1 ); + __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), neg1 ); + + __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); + __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); + + inputCursor += CONTROL_BLOCK_SIZE; + + { + uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_HALF_LO( false, false ); + DECODE_STEP_HALF_HI( false, false ); + } + + { + uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_END_LO( false, false ); + DECODE_STEP_END_HI( false, false ); + } + } + } + + // Decoding loop with all buffer checks. + { + const uint8_t* inputEarlyEnd; + uint8_t* outputEarlyEnd; + inputEarlyEnd = ( input + inputLength ) - END_PADDING_LITERALS; + outputEarlyEnd = ( output + outputLength ) - END_PADDING_LITERALS; + + while ( outputCursor < outputEarlyEnd && inputCursor < inputEarlyEnd ) + { + __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); + __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); + __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); + + __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); + __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); + __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); + + previousCarryHi = carryHi; + + __m128i literalsPerControl = _mm_add_epi8( offsetSize, offsetSize ); + __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); + __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); + + __m128i bytesOutLo = _mm_sub_epi8( controlLo, _mm_andnot_si128( shiftedCarryHi, isLiteralLo ) ); + __m128i bytesOutHi = _mm_sub_epi8( controlHi, _mm_andnot_si128( carryLo, isLiteralHi ) ); + + __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_blendv_epi8( offsetSize, bytesOutLo, isLiteralLo ) ); + __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_blendv_epi8( offsetSize, bytesOutHi, isLiteralHi ) ); + + __m128i neg1 = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); + + __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), neg1 ); + __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), neg1 ); + + __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); + __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); + + inputCursor += CONTROL_BLOCK_SIZE; + + if ( inputCursor > inputEarlyEnd ) + goto BUFFER_END; + + { + uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_HALF_LO( true, true ); + DECODE_STEP_HALF_HI( true, true ); + } + + { + // Now the top halves. + uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_END_LO( true, true ); + DECODE_STEP_END_HI( true, true ); + } + } + +BUFFER_END: + + // When we get here, we have either advanced the right amount on both cursors + // or something bad happened, so leave it as is, so we can tell where + // the error happened. + if ( inputCursor == inputEarlyEnd && outputCursor == outputEarlyEnd ) + { + size_t remainingLiterals = ( input + inputLength ) - inputCursor; + + // copy any trailing literals + memcpy( outputCursor, inputCursor, remainingLiterals ); + + outputCursor += remainingLiterals; + } + } + +MATCH_UNDERFLOW: + + return outputCursor - output; +} diff --git a/contrib/lzsse/lzsse4/lzsse4.h b/contrib/lzsse/lzsse4/lzsse4.h new file mode 100644 index 00000000000..0fa5e6c8ed0 --- /dev/null +++ b/contrib/lzsse/lzsse4/lzsse4.h @@ -0,0 +1,117 @@ +/* +Copyright (c) 2016, Conor Stokes +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef LZSSE4_H__ +#define LZSSE4_H__ + +#pragma once + +/* LZSSE4 - x64/SSE targeted codec for better performance with lower compression ratio data/less optimal compressors. + * Supports minimum 4 byte matches, maximum 15 bytes of match per control word and 4 byte literal runs per control word. + */ + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* Re-usable parse state object for compression. */ +typedef struct LZSSE4_FastParseState LZSSE4_FastParseState; + +/* Allocate the parse state for compression - returns null on failure */ +LZSSE4_FastParseState* LZSSE4_MakeFastParseState(); + +/* De-allocate the parse state for compression */ +void LZSSE4_FreeFastParseState( LZSSE4_FastParseState* toFree ); + +/* Re-usable parse state object for compression. */ +typedef struct LZSSE4_OptimalParseState LZSSE4_OptimalParseState; + +/* Allocate the parse state for compression - returns null on failure. Note + Buffersize has to be greater or equal to any inputLength used with LZSSE2_CompressOptimalParse */ +LZSSE4_OptimalParseState* LZSSE4_MakeOptimalParseState( size_t bufferSize ); + +/* De-allocate the parse state for compression */ +void LZSSE4_FreeOptimalParseState( LZSSE4_OptimalParseState* toFree ); + + +/* "Optimal" compression routine. +* Will compress data into LZSSE4 format, uses hash BST matching to find matches and run an optimal parse (high relative memory usage). Requires SSE 4.1. +* state : Contains the hash table for matching, passed as a parameter so that allocations can be re-used. +* input : Buffer containing uncompressed data to be compressed. May not be null. +* inputLength : Length of the compressed data in the input buffer - note should be under 2GB. +* output : Buffer that will receive the compressed output. +* outputLength : The length reserved in the buffer for compressed data. This should be at least inputLength. Note, +* The compressed data should never be longer than inputLength, as in this case the data is stored raw. +* level : The compression level to use for this file 1->17, 17 delivers the highest compression, 1 delivers the least. +* Thread Safety - state can not be used on multiple threads with calls running concurrently. Can run multiple threads with separate state +* concurrently. +* +* Returns the size of the compressed data, or 0 in the case of error (e.g. outputLength is less than inputLength). +*/ +size_t LZSSE4_CompressOptimalParse( LZSSE4_OptimalParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength, unsigned int level ); + + +/* "Fast" compression routine. + * Will compress data into LZSSE4 format, uses a simple single entry hash/greedy matching to find matches. Requires SSE 4.1. + * state : Contains the hash table for matching, passed as a parameter so that allocations can be re-used. + * input : Buffer containing uncompressed data to be compressed. May not be null. + * inputLength : Length of the compressed data in the input buffer - note should be under 2GB. + * output : Buffer that will receive the compressed output. + * outputLength : The length reserved in the buffer for compressed data. This should be at least inputLength. Note, + * The compressed data should never be longer than inputLength, as in this case the data is stored raw. + * + * Thread Safety - state can not be used on multiple threads with calls running concurrently. Can run multiple threads with separate state + * concurrently. + * + * Returns the size of the compressed data, or 0 in the case of error (e.g. outputLength is less than inputLength). + */ +size_t LZSSE4_CompressFast( LZSSE4_FastParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength ); + +/* Decompression routine. + * This routine will decompress data in the LZSSE4 format and currently requires SSE 4.1 and is targeted at x64. + * It will perform poorly on x86 due to hunger for registers. + * input : Buffer containing compressed input block. May not be null. + * inputLength : Length of the compressed data in the input buffer - note, this should be under 2GB + * output : Buffer that will received the de-compressed output. Note, that this needs to be at least outputLength long. + * May not be null. + * outputLength : The length of the compressed output - note, this should be under 2GB + * + * Provided that input and output are valid pointers to buffers of at least their specified size, this routine + * should be memory safe - both match pointer checks and input/output buffer checks exist. + * + * Returns the size of the decompressed data, which will be less than outputLength in the event of an error (number of bytes + * will indicate where in the output stream the error occured). + * + * Note that this data is not hash verified, errors that occur are either from a misformed stream or bad buffer sizes. + * Remember, corrupt data can still be valid to decompress. + */ +size_t LZSSE4_Decompress( const void* input, size_t inputLength, void* output, size_t outputLength ); + +#ifdef __cplusplus +} +#endif + +#endif /* -- LZSSE4_H__ */ diff --git a/contrib/lzsse/lzsse4/lzsse4_platform.h b/contrib/lzsse/lzsse4/lzsse4_platform.h new file mode 100644 index 00000000000..d1d3f1cced2 --- /dev/null +++ b/contrib/lzsse/lzsse4/lzsse4_platform.h @@ -0,0 +1,73 @@ +/* +Copyright (c) 2016, Brian Marshall +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef LZSSE4_PLATFORM_H__ +#define LZSSE4_PLATFORM_H__ + +#pragma once + +/* + Compiler/Platform detection based on the table from: + https://blogs.msdn.microsoft.com/vcblog/2015/12/04/clang-with-microsoft-codegen-in-vs-2015-update-1/ +*/ + +#ifdef _MSC_VER + +/* + Microsoft Visual Studio Support. + C1xx/C2, Clang/C2 and Clang/LLVM all support the Microsoft header files and _BitScanForward + + Note: if you receive errors with the intrinsics make sure that you have SSE4.1 support enabled. + For example with Clang include "-msse4.1" on the command line +*/ +#include + +#else /* _MSC_VER */ + +#ifdef __GNUC__ + +/* + GCC +*/ + +/* + Note: including just would be sufficient, but including x86intrin is a better match to intrin.h on Visual Studio as + both include all intrinsics for the enabled processor, rather than just SSE4.1. +*/ +#include +/* _BitScanForward is Visual Studio specific. */ +#define _BitScanForward(x, m) *(x) = __builtin_ctz(m) + +#else + +/* +If you hit the error below, then add detection for your compiler/platform to this header file. +*/ +#error Platform not supported + +#endif /* __GNUC__ */ +#endif /* _MSC_VER */ + +#endif /* -- LZSSE4_PLATFORM_H__ */ diff --git a/contrib/lzsse/lzsse8/lzsse8.cpp b/contrib/lzsse/lzsse8/lzsse8.cpp new file mode 100644 index 00000000000..a5682a1de81 --- /dev/null +++ b/contrib/lzsse/lzsse8/lzsse8.cpp @@ -0,0 +1,1568 @@ +/* +Copyright (c) 2016, Conor Stokes +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include +#include +#include +#include + +#include "lzsse8_platform.h" +#include "lzsse8.h" + +#pragma warning ( disable : 4127 ) + +namespace +{ + // Constants - most of these should not be changed without corresponding code changes because it will break many things in unpredictable ways. + const uint32_t WINDOW_BITS = 16; + const uint32_t MIN_MATCH_LENGTH = 4; + const uint32_t LZ_WINDOW_SIZE = 1 << WINDOW_BITS; + const uint32_t LZ_WINDOW_MASK = LZ_WINDOW_SIZE - 1; + const uint32_t FAST_HASH_BITS = 20; // You can change this - more bits = more matches, less bits = more cache hits + const uint32_t FAST_BUCKETS_COUNT = 1 << FAST_HASH_BITS; + const uint32_t FAST_HASH_MASK = FAST_BUCKETS_COUNT - 1; + const uint32_t MIN_COMPRESSION_SIZE = 32; + const uint32_t END_PADDING_LITERALS = 16; + const int32_t NO_MATCH = -1; + const int32_t EMPTY_NODE = -1; + const uint32_t CONTROL_BITS = 4; + const uint32_t OFFSET_SIZE = 2; + const uint32_t EXTENDED_MATCH_BOUND = ( 1 << CONTROL_BITS ) - 1; + const uint32_t CONTROL_BLOCK_SIZE = sizeof( __m128i ); + const uint32_t CONTROLS_PER_BLOCK = 32; + const uint32_t LITERALS_PER_CONTROL = 8; + const uint32_t MAX_INPUT_PER_CONTROL = 8; + const uint32_t INITIAL_MATCH_BOUND = 11; + const size_t OUTPUT_BUFFER_SAFE = EXTENDED_MATCH_BOUND * CONTROLS_PER_BLOCK; + const size_t INPUT_BUFFER_SAFE = MAX_INPUT_PER_CONTROL * CONTROLS_PER_BLOCK; + const uint16_t INITIAL_OFFSET = LITERALS_PER_CONTROL; + const uint32_t OFFSET_BITS = 16; + const uint32_t BASE_MATCH_BITS = OFFSET_BITS + CONTROL_BITS; + const uint32_t OPTIMAL_HASH_BITS = 20; + const uint32_t OPTIMAL_BUCKETS_COUNT = 1 << OPTIMAL_HASH_BITS; + const uint32_t OPTIMAL_HASH_MASK = OPTIMAL_BUCKETS_COUNT - 1; + const uint32_t LITERAL_BITS = 8; + const size_t SKIP_MATCH_LENGTH = 128; + const uint32_t NO_SKIP_LEVEL = 17; + +} + + +struct LZSSE8_FastParseState +{ + int32_t buckets[ FAST_BUCKETS_COUNT ]; // stores the first matching position, we can then look at the rest of the matches by tracing through the window. +}; + + +LZSSE8_FastParseState* LZSSE8_MakeFastParseState() +{ + return new LZSSE8_FastParseState(); +} + + +void LZSSE8_FreeFastParseState( LZSSE8_FastParseState* toFree ) +{ + delete toFree; +} + + +inline void SetHash( LZSSE8_FastParseState* state, uint32_t hash, const uint8_t* input, const uint8_t* inputCursor ) +{ + int32_t position = static_cast( inputCursor - input ); + + state->buckets[ hash & FAST_HASH_MASK ] = position; +} + +// Simple fast hash function - actually what is used in snappy and derivatives +// There's probably better, but I haven't spent time focusing on this area yet. +inline uint32_t HashFast( const uint8_t* inputCursor ) +{ + return *reinterpret_cast( inputCursor ) * 0x1e35a7bd >> ( 32 - FAST_HASH_BITS ); +} + +struct Arrival +{ + size_t cost; + int32_t from; + int32_t to; + uint16_t offset; +}; + +struct TreeNode +{ + int32_t children[ 2 ]; +}; + +struct LZSSE8_OptimalParseState +{ + int32_t roots[ OPTIMAL_BUCKETS_COUNT ]; + + TreeNode window[ LZ_WINDOW_SIZE ]; + + Arrival* arrivals; + + size_t bufferSize; +}; + + +LZSSE8_OptimalParseState* LZSSE8_MakeOptimalParseState( size_t bufferSize ) +{ + if ( bufferSize > 0 && ( SIZE_MAX / sizeof( Arrival ) ) < bufferSize ) + { + return nullptr; + } + + LZSSE8_OptimalParseState* result = reinterpret_cast< LZSSE8_OptimalParseState* >( ::malloc( sizeof( LZSSE8_OptimalParseState ) ) ); + + result->bufferSize = bufferSize; + + if ( result != nullptr ) + { + result->arrivals = reinterpret_cast< Arrival* >( ::malloc( sizeof( Arrival ) * bufferSize ) ); + + if ( result->arrivals == nullptr ) + { + LZSSE8_FreeOptimalParseState( result ); + + result = nullptr; + } + } + + return result; +} + + +void LZSSE8_FreeOptimalParseState( LZSSE8_OptimalParseState* toFree ) +{ + ::free( toFree->arrivals ); + + toFree->arrivals = nullptr; + + ::free( toFree ); +} + + +inline uint32_t HashOptimal( const uint8_t* inputCursor ) +{ + return *reinterpret_cast( inputCursor ) * 0x1e35a7bd >> ( 32 - OPTIMAL_HASH_BITS ); +} + + +struct Match +{ + size_t length; + int32_t position; + uint16_t offset; +}; + + +inline Match SearchAndUpdateFinder( LZSSE8_OptimalParseState& state, const uint8_t* input, const uint8_t* inputCursor, const uint8_t* inputEnd, uint32_t cutOff ) +{ + Match result; + + int32_t position = static_cast( inputCursor - input ); + + result.position = NO_MATCH; + result.length = MIN_MATCH_LENGTH; + result.offset = 0; + + size_t lengthToEnd = inputEnd - inputCursor; + int32_t lastPosition = position - ( LZ_WINDOW_SIZE - 1 ); + uint32_t hash = HashOptimal( inputCursor ); + + lastPosition = lastPosition > 0 ? lastPosition : 0; + + int32_t treeCursor = state.roots[ hash ]; + + state.roots[ hash ] = position; + + int32_t* left = &state.window[ position & LZ_WINDOW_MASK ].children[ 1 ]; + int32_t* right = &state.window[ position & LZ_WINDOW_MASK ].children[ 0 ]; + size_t leftLength = 0; + size_t rightLength = 0; + + for ( ;; ) + { + if ( cutOff-- == 0 || treeCursor < lastPosition ) + { + *left = *right = EMPTY_NODE; + break; + } + + TreeNode& currentNode = state.window[ treeCursor & LZ_WINDOW_MASK ]; + const uint8_t* key = input + treeCursor; + size_t matchLength = leftLength < rightLength ? leftLength : rightLength; + + uint16_t matchOffset = static_cast( position - treeCursor ); + size_t maxLength = matchOffset <= ( EXTENDED_MATCH_BOUND + 1 ) && matchOffset < lengthToEnd ? matchOffset : lengthToEnd; + + while ( matchLength < lengthToEnd ) + { + __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); + __m128i match16 = _mm_loadu_si128( reinterpret_cast( key + matchLength ) ); + + unsigned long matchBytes; + + _BitScanForward( &matchBytes, ( static_cast( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); + + matchLength += matchBytes; + + if ( matchBytes != 16 ) + { + break; + } + } + + matchLength = matchLength < lengthToEnd ? matchLength : lengthToEnd; + + size_t truncatedMatchLength = matchLength < maxLength ? matchLength : maxLength; + + if ( truncatedMatchLength >= result.length && matchOffset >= LITERALS_PER_CONTROL ) + { + result.length = truncatedMatchLength; + result.offset = matchOffset; + result.position = treeCursor; + } + + if ( matchLength == lengthToEnd ) + { + *left = currentNode.children[ 1 ]; + *right = currentNode.children[ 0 ]; + break; + } + + if ( inputCursor[ matchLength ] < key[ matchLength ] || ( matchLength == lengthToEnd ) ) + { + *left = treeCursor; + left = currentNode.children; + treeCursor = *left; + leftLength = matchLength; + } + else + { + *right = treeCursor; + right = currentNode.children + 1; + treeCursor = *right; + rightLength = matchLength; + } + } + + // Special RLE overlapping match case, the LzFind style match above doesn't work very well with our + // restriction of overlapping matches having offsets of at least 16. + // Suffix array seems like a better option to handling this. + { + // Note, we're detecting long RLE here, but if we have an offset too close, we'll sacrifice a fair + // amount of decompression performance to load-hit-stores. + int32_t matchPosition = position - ( sizeof( __m128i ) * 2 ); + + if ( matchPosition >= 0 ) + { + uint16_t matchOffset = static_cast( position - matchPosition ); + const uint8_t* key = input + matchPosition; + size_t matchLength = 0; + + while ( matchLength < lengthToEnd ) + { + __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); + __m128i match16 = _mm_loadu_si128( reinterpret_cast( key + matchLength ) ); + + unsigned long matchBytes; + + _BitScanForward( &matchBytes, ( static_cast( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); + + matchLength += matchBytes; + + if ( matchBytes != 16 ) + { + break; + } + + } + + matchLength = matchLength < lengthToEnd ? matchLength : lengthToEnd; + + if ( matchLength >= result.length ) + { + result.length = matchLength; + result.offset = matchOffset; + result.position = matchPosition; + } + } + } + + return result; +} + + +size_t LZSSE8_CompressOptimalParse( LZSSE8_OptimalParseState* state, const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength, unsigned int level ) +{ + if ( outputLength < inputLength || state->bufferSize < inputLength ) + { + // error case, output buffer not large enough. + return 0; + } + + const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); + uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); + + if ( inputLength < MIN_COMPRESSION_SIZE ) + { + memcpy( output, input, inputLength ); + + return inputLength; + } + + const uint8_t* inputCursor = input; + const uint8_t* inputEnd = input + inputLength; + Arrival* arrivalWatermark = state->arrivals; + Arrival* arrival = state->arrivals; + uint32_t cutOff = 1 << level; + + for ( int32_t* rootCursor = state->roots, *end = rootCursor + OPTIMAL_BUCKETS_COUNT; rootCursor < end; rootCursor += 4 ) + { + rootCursor[ 0 ] = EMPTY_NODE; + rootCursor[ 1 ] = EMPTY_NODE; + rootCursor[ 2 ] = EMPTY_NODE; + rootCursor[ 3 ] = EMPTY_NODE; + } + + for ( uint32_t where = 0; where < LITERALS_PER_CONTROL; ++where ) + { + SearchAndUpdateFinder( *state, input, inputCursor, inputEnd - END_PADDING_LITERALS, cutOff ); + + ++inputCursor; + } + + arrival->cost = LITERAL_BITS * LITERALS_PER_CONTROL; + arrival->from = -1; + arrival->offset = 0; + + // loop through each character and project forward the matches at that character to calculate the cheapest + // path of arrival for each individual character. + for ( const uint8_t* earlyEnd = inputEnd - END_PADDING_LITERALS; inputCursor < earlyEnd; ++inputCursor, ++arrival ) + { + uint32_t lengthToEnd = static_cast< uint32_t >( earlyEnd - inputCursor ); + int32_t currentPosition = static_cast< int32_t >( inputCursor - input ); + size_t literalsForward = LITERALS_PER_CONTROL < lengthToEnd ? LITERALS_PER_CONTROL : lengthToEnd; + size_t arrivalCost = arrival->cost; + + // NOTE - we currently assume only 2 literals filled in here, because the minimum match length is 3. + // If we wanted to go with a higher minimum match length, we would need to fill in more literals before hand. + // Also, because there is a maximum of 2 literals per control block assumed. + + // project forward the cost of a single literal + + for ( size_t where = 1; where <= literalsForward; ++where ) + { + Arrival* literalArrival = arrival + where; + size_t literalCost = arrivalCost + CONTROL_BITS + ( where * LITERAL_BITS ); + + if ( literalArrival > arrivalWatermark || literalArrival->cost > literalCost ) + { + literalArrival->cost = literalCost; + literalArrival->from = currentPosition; + literalArrival->offset = 0; + + arrivalWatermark = literalArrival > arrivalWatermark ? literalArrival : arrivalWatermark; + } + } + + Match match = SearchAndUpdateFinder( *state, input, inputCursor, earlyEnd, cutOff ); + + if ( match.position != NO_MATCH ) + { + for ( size_t matchedLength = MIN_MATCH_LENGTH, end = match.length + 1; matchedLength < end; ++matchedLength ) + { + Arrival* matchArrival = arrival + matchedLength; + size_t matchCost = arrivalCost + BASE_MATCH_BITS; + + if ( matchedLength >= INITIAL_MATCH_BOUND ) + { + matchCost += ( ( ( matchedLength - INITIAL_MATCH_BOUND ) / EXTENDED_MATCH_BOUND ) + 1 ) * CONTROL_BITS; + } + + if ( matchArrival > arrivalWatermark || matchArrival->cost > matchCost ) + { + matchArrival->cost = matchCost; + matchArrival->from = currentPosition; + matchArrival->offset = match.offset; + + arrivalWatermark = matchArrival > arrivalWatermark ? matchArrival : arrivalWatermark; + } + } + + if ( match.length > SKIP_MATCH_LENGTH && level < NO_SKIP_LEVEL ) + { + arrival += match.length - LITERALS_PER_CONTROL; + inputCursor += match.length - LITERALS_PER_CONTROL; + } + } + } + + // If this would cost more to encode than it would if it were just literals, encode it with no control blocks, + // just literals + if ( ( arrivalWatermark->cost + END_PADDING_LITERALS * LITERAL_BITS + CONTROLS_PER_BLOCK * CONTROL_BITS ) > ( inputLength * LITERAL_BITS ) ) + { + memcpy( output, input, inputLength ); + + return inputLength; + } + + Arrival* previousPathNode; + + // now trace the actual optimal parse path back, connecting the nodes in the other direction. + for ( const Arrival* pathNode = arrivalWatermark; pathNode->from > 0; pathNode = previousPathNode ) + { + previousPathNode = state->arrivals + ( pathNode->from - LITERALS_PER_CONTROL ); + + previousPathNode->to = static_cast( ( pathNode - state->arrivals ) + LITERALS_PER_CONTROL ); + } + + uint8_t* outputCursor = output; + + memcpy( outputCursor, input, LITERALS_PER_CONTROL ); + + outputCursor += LITERALS_PER_CONTROL; + + uint8_t* currentControlBlock = outputCursor; + uint32_t currentControlCount = 0; + uint32_t totalControlCount = 0; + + outputCursor += CONTROL_BLOCK_SIZE; + + Arrival* nextPathNode; + + size_t totalPathLength = LITERALS_PER_CONTROL; + uint16_t previousOffset = INITIAL_OFFSET; + + bool lastControlIsNop = false; + + // Now walk forwards again and actually write out the data. + for ( const Arrival* pathNode = state->arrivals; pathNode < arrivalWatermark; pathNode = nextPathNode ) + { + int32_t currentPosition = static_cast< int32_t >( ( pathNode - state->arrivals ) + LITERALS_PER_CONTROL ); + + nextPathNode = state->arrivals + ( pathNode->to - LITERALS_PER_CONTROL ); + + size_t pathDistance = nextPathNode - pathNode; + + totalPathLength += pathDistance; + + lastControlIsNop = false; + + if ( nextPathNode->offset == 0 ) + { + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + currentControlCount = 0; + } + + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + ( static_cast( pathDistance ) - 1 ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + ( static_cast< uint8_t >( pathDistance ) - 1 ) << CONTROL_BITS; + } + + // output the literals. + for ( int32_t where = 0; where < pathDistance; ++where ) + { + const uint8_t* currentInput = input + currentPosition + where; + + outputCursor[ where ] = *currentInput ^ *( currentInput - previousOffset ); + } + + outputCursor += pathDistance; + + ++totalControlCount; + ++currentControlCount; + } + else + { + // Check if we need to start a new control block + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + } + + // Write the offset out - note the xor with the previous offset. + *reinterpret_cast< uint16_t* >( outputCursor ) = nextPathNode->offset ^ previousOffset; + + previousOffset = nextPathNode->offset; + outputCursor += sizeof( uint16_t ); + + if ( pathDistance < INITIAL_MATCH_BOUND ) + { + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + static_cast( pathDistance + MIN_MATCH_LENGTH ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + static_cast< uint8_t >( pathDistance + MIN_MATCH_LENGTH ) << CONTROL_BITS; + } + + ++currentControlCount; + } + else + { + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + static_cast( EXTENDED_MATCH_BOUND ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + static_cast< uint8_t >( EXTENDED_MATCH_BOUND ) << CONTROL_BITS; + } + + ++currentControlCount; + + size_t toEncode = pathDistance - INITIAL_MATCH_BOUND; + + for ( ;; ) + { + // Check if we need to start a new control block + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + } + + // If the encode size is greater than we can hold in a control, write out a full match length + // control, subtract full control value from the amount to encode and loop around again. + if ( toEncode >= EXTENDED_MATCH_BOUND ) + { + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + static_cast( EXTENDED_MATCH_BOUND ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + static_cast< uint8_t >( EXTENDED_MATCH_BOUND ) << CONTROL_BITS; + } + + toEncode -= EXTENDED_MATCH_BOUND; + + ++currentControlCount; + } + else // Write out the remaining match length control. Could potentially be zero. + { + if ( ( currentControlCount & 1 ) == 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] = + static_cast( toEncode ); + } + else + { + currentControlBlock[ currentControlCount >> 1 ] |= + static_cast< uint8_t >( toEncode ) << CONTROL_BITS; + } + + if ( toEncode == 0 && currentControlCount == 0 ) + { + lastControlIsNop = true; + } + + ++currentControlCount; + + break; + } + } + } + } + } + + if ( lastControlIsNop ) + { + outputCursor -= CONTROL_BLOCK_SIZE; + } + + size_t remainingLiterals = ( input + inputLength ) - inputCursor; + + // copy remaining literals + memcpy( outputCursor, inputCursor, remainingLiterals ); + + outputCursor += remainingLiterals; + + return outputCursor - output; +} + + +size_t LZSSE8_CompressFast( LZSSE8_FastParseState* state, const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength ) +{ + if ( outputLength < inputLength ) + { + // error case, output buffer not large enough. + return 0; + } + + const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); + uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); + + if ( inputLength < MIN_COMPRESSION_SIZE ) + { + memcpy( output, input, inputLength ); + + return inputLength; + } + + const uint8_t* inputCursor = input; + const uint8_t* inputEnd = input + inputLength; + const uint8_t* inputEarlyEnd = inputEnd - END_PADDING_LITERALS; + uint8_t* outputCursor = output; + uint8_t* outputEarlyEnd = ( output + outputLength ) - END_PADDING_LITERALS; + uint32_t hash = 0; + + // initialize hash to empty + for ( int32_t* where = state->buckets, *end = state->buckets + FAST_BUCKETS_COUNT; where < end; where += 4 ) + { + where[ 0 ] = -1; + where[ 1 ] = -1; + where[ 2 ] = -1; + where[ 3 ] = -1; + } + + // initial literals that wont be compressed + for ( uint32_t where = 0; where < LITERALS_PER_CONTROL; ++where ) + { + hash = HashFast( inputCursor ); + + SetHash( state, hash, input, inputCursor ); + + *( outputCursor++ ) = *( inputCursor++ ); + } + + uint8_t* currentControlBlock = outputCursor; + uint32_t currentControlCount = 0; + uint16_t previousOffset = INITIAL_OFFSET; + size_t literalsToFlush = 0; + + outputCursor += CONTROL_BLOCK_SIZE; + + bool lastControlIsNop = false; + + // Loop through the data until we hit the end of one of the buffers (minus the end padding literals) + while ( inputCursor < inputEarlyEnd && outputCursor <= outputEarlyEnd ) + { + lastControlIsNop = false; + + hash = HashFast( inputCursor ); + + int matchPosition = state->buckets[ hash & FAST_HASH_MASK ]; + int currentPosition = static_cast< int32_t >( inputCursor - input ); + uint32_t matchLength = 0; + uint16_t matchOffset = static_cast< uint16_t >( currentPosition - matchPosition ); + + // If we had a hit in the hash and it wasn't outside the window. + if ( matchPosition >= 0 && ( currentPosition - matchPosition ) < ( LZ_WINDOW_SIZE - 1 ) && matchOffset >= LITERALS_PER_CONTROL ) + { + const uint8_t* matchCandidate = input + matchPosition; + uint32_t lengthToEnd = static_cast< uint32_t >( inputEarlyEnd - inputCursor ); + // Here we limit the hash length to prevent overlap matches with offset less than 16 bytes + uint32_t maxLength = matchOffset <= ( EXTENDED_MATCH_BOUND + 1 ) && matchOffset < lengthToEnd ? matchOffset : lengthToEnd; + + // Find how long the match is 16 bytes at a time. + while ( matchLength < maxLength ) + { + __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); + __m128i match16 = _mm_loadu_si128( reinterpret_cast( matchCandidate + matchLength ) ); + + unsigned long matchBytes; + + // Finds the number of equal bytes at the start of the 16 + _BitScanForward( &matchBytes, ( static_cast< unsigned long >( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); + + matchLength += matchBytes; + + if ( matchBytes != sizeof( __m128i ) ) + { + break; + } + } + + matchLength = matchLength < maxLength ? matchLength : maxLength; + } + + // If we have at least the minimum match length (4 bytes) + if ( matchLength >= MIN_MATCH_LENGTH ) + { + // Do we have literals to flush before the match? + if ( literalsToFlush > 0 ) + { + // Start a new control block if we need one. + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + + // Would be larger than compressed size, get out! + if ( outputCursor > outputEarlyEnd ) + { + break; + } + } + + currentControlBlock[ currentControlCount >> 1 ] = + ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( literalsToFlush - 1 ) << 4 ); + + // flush the literals. + // note the xor against the data that would be read in the match. + for ( uint32_t where = 0; where < literalsToFlush; ++where ) + { + const uint8_t* currentInput = inputCursor - ( literalsToFlush - where ); + + *( outputCursor++ ) = *currentInput ^ *( currentInput - previousOffset ); + } + + ++currentControlCount; + + literalsToFlush = 0; + + // Would be larger than compressed size, get out! + if ( outputCursor > outputEarlyEnd ) + { + break; + } + } + + // Start a new control block if the previous one is full. + // Note this is done before the offset is written out - the offset + // is always written after the control block containing the first + // control in the match. + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + + if ( outputCursor > outputEarlyEnd ) + { + break; + } + } + + // The match length value we are encoding. + + // Write the offset out - note the xor with the previous offset. + *reinterpret_cast< uint16_t* >( outputCursor ) = matchOffset ^ previousOffset; + + previousOffset = matchOffset; + outputCursor += sizeof( uint16_t ); + + if ( matchLength < INITIAL_MATCH_BOUND ) + { + currentControlBlock[ currentControlCount >> 1 ] = + ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( matchLength + MIN_MATCH_LENGTH ) << 4 ); + + ++currentControlCount; + } + else + { + currentControlBlock[ currentControlCount >> 1 ] = + ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( EXTENDED_MATCH_BOUND ) << 4 ); + + ++currentControlCount; + + size_t toEncode = matchLength - INITIAL_MATCH_BOUND; + + for ( ;; ) + { + // Check if we need to start a new control block + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + + if ( outputCursor > outputEarlyEnd ) + { + break; + } + } + + // If the encode size is greater than we can hold in a control, write out a full match length + // control, subtract full control value from the amount to encode and loop around again. + if ( toEncode >= EXTENDED_MATCH_BOUND ) + { + currentControlBlock[ currentControlCount >> 1 ] = + ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( EXTENDED_MATCH_BOUND ) << 4 ); + + toEncode -= EXTENDED_MATCH_BOUND; + + ++currentControlCount; + } + else // Write out the remaining match length control. Could potentially be zero. + { + currentControlBlock[ currentControlCount >> 1 ] = + ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( toEncode ) << 4 ); + + if ( currentControlBlock == 0 && toEncode == 0 ) + { + lastControlIsNop = true; + } + + ++currentControlCount; + + break; + } + } + } + + // Update the value into the hash for future matches. + SetHash( state, hash, input, inputCursor ); + + ++inputCursor; + + // Hash all the other values in the match too. + for ( const uint8_t* nextArrival = inputCursor + matchLength - 1; inputCursor < nextArrival; ++inputCursor ) + { + hash = HashFast( inputCursor ); + SetHash( state, hash, input, inputCursor ); + } + } + else + { + // One more literal to write out. + ++literalsToFlush; + + // If we have reached the maximum number of literals allowed in the control, flush them out. + if ( literalsToFlush == LITERALS_PER_CONTROL ) + { + // Check if the control block is full and we need start a new one. + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + + if ( outputCursor > outputEarlyEnd ) + { + break; + } + } + + currentControlBlock[ currentControlCount >> 1 ] = + ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( ( static_cast( LITERALS_PER_CONTROL - 1 ) ) << 4 ); + + ++currentControlCount; + + *reinterpret_cast< uint64_t* >( outputCursor ) = + *reinterpret_cast< const uint64_t* >( inputCursor - ( sizeof( uint64_t ) - 1 ) ) ^ + *reinterpret_cast< const uint64_t* >( ( inputCursor - ( sizeof( uint64_t ) - 1 ) ) - previousOffset ); + + outputCursor += sizeof( uint64_t ); + + //*( outputCursor++ ) = *( inputCursor - 3 ) ^ *( ( inputCursor - 3 ) - previousOffset ); + //*( outputCursor++ ) = *( inputCursor - 2 ) ^ *( ( inputCursor - 2 ) - previousOffset ); + //*( outputCursor++ ) = *( inputCursor - 1 ) ^ *( ( inputCursor - 1 ) - previousOffset ); + //*( outputCursor++ ) = *inputCursor ^ *( inputCursor - previousOffset ); + + if ( outputCursor > outputEarlyEnd ) + { + break; + } + + literalsToFlush = 0; + } + + // Update the hash with this byte + SetHash( state, hash, input, inputCursor ); + + ++inputCursor; + } + } + + // If we would create a compression output bigger than or equal to the input, just copy the input to the output and return equal size. + if ( ( ( outputCursor + literalsToFlush + ( currentControlCount == CONTROLS_PER_BLOCK ? CONTROL_BLOCK_SIZE : 0 ) ) ) >= output + inputLength - END_PADDING_LITERALS ) + { + memcpy( output, input, inputLength ); + + outputCursor = output + inputLength; + } + else + { + // Flush any remaining literals. + if ( literalsToFlush > 0 ) + { + lastControlIsNop = false; + + if ( currentControlCount == CONTROLS_PER_BLOCK ) + { + currentControlBlock = outputCursor; + outputCursor += CONTROL_BLOCK_SIZE; + + _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); + + currentControlCount = 0; + } + + currentControlBlock[ currentControlCount >> 1 ] = + ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( literalsToFlush - 1 ) << 4 ); + + for ( uint32_t where = 0; where < literalsToFlush; ++where ) + { + const uint8_t* currentInput = inputCursor - ( literalsToFlush - where ); + + *( outputCursor++ ) = *currentInput ^ *( currentInput - previousOffset ); + } + + ++currentControlCount; + } + + // Need to finish off shifting the final control block into the low nibble if there is no second nibble + if ( ( currentControlCount & 1 ) > 0 ) + { + currentControlBlock[ currentControlCount >> 1 ] >>= 4; + } + + if ( lastControlIsNop ) + { + outputCursor -= CONTROL_BLOCK_SIZE; + } + + size_t remainingLiterals = ( input + inputLength ) - inputCursor; + + // copy remaining literals + memcpy( outputCursor, inputCursor, remainingLiterals ); + + outputCursor += remainingLiterals; + } + + // Return the size of the compressed data. + return outputCursor - output; +} + + +size_t LZSSE8_Decompress( const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength ) +{ + const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); + uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); + + // Data was not compressible, just copy initial values + if ( outputLength == inputLength ) + { + memcpy( output, input, outputLength ); + + return inputLength; + } + + const uint8_t* inputCursor = input; + uint8_t* outputCursor = output; + + // The offset starts off as the minimum match length. We actually need it least four + // characters back because we need them to be set to xor out the literals from the match data. + size_t offset = INITIAL_OFFSET; + __m128i previousCarryHi = _mm_setzero_si128(); + + // Copy the initial literals to the output. + for ( uint32_t where = 0; where < LITERALS_PER_CONTROL; ++where ) + { + *( outputCursor++ ) = *( inputCursor++ ); + } + + // Let me be clear, I am usually anti-macro, but they work for this particular (very unusual) case. + // DECODE_STEP is a regular decoding step, DECODE_STEP_HALF and DECODE_STEP_END are because the compiler couldn't + // seem to remove some of the dead code where values were updated and then never used. + + // What these macros do: + // Decode a single literal run or match run for a single control nibble. + // How they do it: + // - Read the *unaligned* input (in the case of LZSSE-F - twice), it goes into both a regular variable and an SSE register, + // because it could either be literals or an offset (or nothing at all). The low byte of streamBytesRead controls how much we advance + // the input cursor. + // - Used a contived set of casts to sign extend the "read offset" control mask and then use it to mask the input word, + // which is then xor'd against the offset, for a "branchless" conditional move into the offset which + // has been carried over from the previous literal/match block. Note, this ends up doing better than a cmov on most + // modern processors. But we need to pre-xor the input offset. + // - We then load the match data from output buffer (offset back from the current output point). Unconditional load here. + // - We broadcast the "from literal" control mask from the current least significant byte of the SSE register using a shuffle epi-8 + // - We mask the literals with that SSE register wide mask. + // - The literals have been pre-xor'd with the data read in as match data, so we use an xor to branchlessly choose between the two. + // In this case, it ends up a better option than a blendv on most processors. + // - Store the block. We store all 16 bytes of the SSE register (due to some constraints in the format of the data, we won't + // go past the end of the buffer), but we may overlap this. + // - bytesOut controls how much we advance the output cursor. + // - We use 8 bit shifts to advance all the controls up to the next byte. There is some variable sized register trickery that + // x86/x64 is great for as long as we don't anger the register renamer. + +#define DECODE_STEP( HILO, CHECKMATCH, CHECKBUFFERS ) \ + { \ + size_t inputWord = *reinterpret_cast( inputCursor ); \ + __m128i literals = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); \ + \ + offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ + \ + readOffsetHalf##HILO >>= 8; \ + \ + const uint8_t* matchPointer = reinterpret_cast( outputCursor - offset ); \ + \ + if ( CHECKMATCH && matchPointer < output ) \ + goto MATCH_UNDERFLOW; \ + \ + __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ + __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ + \ + literals = _mm_and_si128( literals, fromLiteral ); \ + \ + fromLiteral##HILO = _mm_srli_si128( fromLiteral##HILO, 1 ); \ + \ + __m128i toStore = _mm_xor_si128( matchData, literals ); \ + \ + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ + \ + outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ + inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ + \ + bytesOutHalf##HILO >>= 8; \ + streamBytesReadHalf##HILO >>= 8; \ + \ + if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ + goto BUFFER_END; \ + } + +#define DECODE_STEP_HALF( HILO, CHECKMATCH, CHECKBUFFERS ) \ + { \ + size_t inputWord = *reinterpret_cast( inputCursor ); \ + __m128i literals = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); \ + \ + offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ + \ + const uint8_t* matchPointer = reinterpret_cast( outputCursor - offset ); \ + \ + if ( CHECKMATCH && matchPointer < output ) \ + goto MATCH_UNDERFLOW; \ + \ + __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ + __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ + \ + literals = _mm_and_si128( literals, fromLiteral ); \ + \ + fromLiteral##HILO = _mm_srli_si128( fromLiteral##HILO, 1 ); \ + \ + __m128i toStore = _mm_xor_si128( matchData, literals ); \ + \ + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ + \ + outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ + inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ + \ + if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ + goto BUFFER_END; \ + } + +#define DECODE_STEP_END( HILO, CHECKMATCH, CHECKBUFFERS ) \ + { \ + size_t inputWord = *reinterpret_cast( inputCursor ); \ + __m128i literals = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); \ + \ + offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ + \ + const uint8_t* matchPointer = reinterpret_cast( outputCursor - offset ); \ + \ + if ( CHECKMATCH && matchPointer < output ) \ + goto MATCH_UNDERFLOW; \ + \ + __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ + __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ + \ + literals = _mm_and_si128( literals, fromLiteral ); \ + \ + __m128i toStore = _mm_xor_si128( matchData, literals ); \ + \ + _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ + \ + outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ + inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ + \ + if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ + goto BUFFER_END; \ + } + +#define DECODE_STEP_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP( Lo, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP( Hi, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_HALF_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_HALF( Lo, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_HALF_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_HALF( Hi, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_END_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_END( Lo, CHECKMATCH, CHECKBUFFERS ) +#define DECODE_STEP_END_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_END( Hi, CHECKMATCH, CHECKBUFFERS ) + + __m128i nibbleMask = _mm_set1_epi8( 0xF ); + __m128i literalsPerControl = _mm_set1_epi8( static_cast< char >( LITERALS_PER_CONTROL ) ); + __m128i bytesInOutLUT = _mm_set_epi8( '\x2B', '\x2A', '\x29', '\x28', '\x27', '\x26', '\x25', '\x24', '\x88', '\x77', '\x66', '\x55', '\x44', '\x33', '\x22', '\x11' ); + + // Note, we use this block here because it allows the "fake" inputEarlyEnd/outputEarlyEnd not to cause register spills + // in the decompression loops. And yes, that did actually happen. + { + +#pragma warning ( push ) +#pragma warning ( disable : 4101 ) + + // These variables are not actually ever used in this block, because we use + // a constant conditional expression to take out the branches that would hit them. + // But unfortunately, we need them to compile. + const uint8_t* inputEarlyEnd; + uint8_t* outputEarlyEnd; + +#pragma warning ( pop ) + + // "Safe" ends to the buffer, before the input/output cursors hit these, we can loop without overflow checks. + const uint8_t* inputSafeEnd = ( input + inputLength ) - INPUT_BUFFER_SAFE; + uint8_t* outputSafeEnd = ( output + outputLength ) - OUTPUT_BUFFER_SAFE; + + // Decoding loop with offset output buffer underflow test, but no buffer overflow tests, assumed to end at a safe distance + // from overflows + while ( ( outputCursor - output ) < LZ_WINDOW_SIZE && outputCursor < outputSafeEnd && inputCursor < inputSafeEnd ) + { + // load the control block + __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); + + // split the control block into high and low nibbles + __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); + __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); + + // Note, the carries are set when the nibble is at its highest value, 15, meaning the operation after will + // be an extension of the current match operation. + + // Work out the carry for the low nibbles (which will be used with the high controls to put them into + // match without offset read mode). + __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); + + // The carry for the high nibbles is used with the low controls, but needs one byte from the previous iteration. We save + // the calculated carry to use that byte next iteration. + __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); + __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); + + previousCarryHi = carryHi; + + // We make the implicit assumption that the maximum number of literals to controls here is twice the offset size (4 vs 2), + // we are doing this here to save keeping the value around (spilling or fetching it each time) + + __m128i streamBytesLo = _mm_shuffle_epi8( bytesInOutLUT, controlLo ); + __m128i streamBytesHi = _mm_shuffle_epi8( bytesInOutLUT, controlHi ); + + // Here we're calculating the number of bytes that will be output, we are actually subtracting negative one from the control + // (handy trick where comparison result masks are negative one) if carry is not set and it is a literal. + __m128i bytesOutLo = _mm_blendv_epi8( _mm_and_si128( streamBytesLo, nibbleMask ), controlLo, shiftedCarryHi ); + __m128i bytesOutHi = _mm_blendv_epi8( _mm_and_si128( streamBytesHi, nibbleMask ), controlHi, carryLo ); + + // Calculate the number of bytes to read per control. + // In the case the carry is set, no bytes. Otherwise, the offset size (2 bytes) for matches or the number of output bytes for literals. + __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_and_si128( _mm_srli_epi32( streamBytesLo, 4 ), nibbleMask ) ); + __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_and_si128( _mm_srli_epi32( streamBytesHi, 4 ), nibbleMask ) ); + + // Here we are testing if the runs will be literals or matches. Note that if the carries are set from the previous operation + // this will essentially be ignored later on. + __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); + __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); + + // I want 128 set bits please. + __m128i allSet = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); + + // Masks to read the offset (or keep the previous one) - set in the case that this is not a literal and the carry is not set + __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), allSet ); + __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), allSet ); + + // Masks whether we are reading literals - set if the carry is not set and these are literals. + __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); + __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); + + // Advance the input past the control block + inputCursor += CONTROL_BLOCK_SIZE; + + { + // Pull out the bottom halves off the SSE registers from before - we want these + // things in GPRs for the more linear logic. + uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_HALF_LO( true, false ); + DECODE_STEP_HALF_HI( true, false ); + } + + { + // Now the top halves. + uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + + DECODE_STEP_LO( true, false ); + DECODE_STEP_HI( true, false ); + DECODE_STEP_END_LO( true, false ); + DECODE_STEP_END_HI( true, false ); + } + } + + // Decoding loop with no buffer checks, but will end at a safe distance from the end of the buffers. + // Note, when we get here we have already reached the point in the output buffer which is *past* where we can underflow + // due to a bad match offset. + while ( outputCursor < outputSafeEnd && inputCursor < inputSafeEnd ) + { + // This code is the same as the loop above, see comments there + + // load the control block + __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); + + // split the control block into high and low nibbles + __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); + __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); + + // Note, the carries are set when the nibble is at its highest value, 15, meaning the operation after will + // be an extension of the current match operation. + + // Work out the carry for the low nibbles (which will be used with the high controls to put them into + // match without offset read mode). + __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); + + // The carry for the high nibbles is used with the low controls, but needs one byte from the previous iteration. We save + // the calculated carry to use that byte next iteration. + __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); + __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); + + previousCarryHi = carryHi; + + // We make the implicit assumption that the maximum number of literals to controls here is twice the offset size (4 vs 2), + // we are doing this here to save keeping the value around (spilling or fetching it each time) + + __m128i streamBytesLo = _mm_shuffle_epi8( bytesInOutLUT, controlLo ); + __m128i streamBytesHi = _mm_shuffle_epi8( bytesInOutLUT, controlHi ); + + // Here we're calculating the number of bytes that will be output, we are actually subtracting negative one from the control + // (handy trick where comparison result masks are negative one) if carry is not set and it is a literal. + __m128i bytesOutLo = _mm_blendv_epi8( _mm_and_si128( streamBytesLo, nibbleMask ), controlLo, shiftedCarryHi ); + __m128i bytesOutHi = _mm_blendv_epi8( _mm_and_si128( streamBytesHi, nibbleMask ), controlHi, carryLo ); + + // Calculate the number of bytes to read per control. + // In the case the carry is set, no bytes. Otherwise, the offset size (2 bytes) for matches or the number of output bytes for literals. + __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_and_si128( _mm_srli_epi32( streamBytesLo, 4 ), nibbleMask ) ); + __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_and_si128( _mm_srli_epi32( streamBytesHi, 4 ), nibbleMask ) ); + + // Here we are testing if the runs will be literals or matches. Note that if the carries are set from the previous operation + // this will essentially be ignored later on. + __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); + __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); + + // I want 128 set bits please. + __m128i allSet = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); + + // Masks to read the offset (or keep the previous one) - set in the case that this is not a literal and the carry is not set + __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), allSet ); + __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), allSet ); + + // Masks whether we are reading literals - set if the carry is not set and these are literals. + __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); + __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); + + inputCursor += CONTROL_BLOCK_SIZE; + + { + uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_HALF_LO( false, false ); + DECODE_STEP_HALF_HI( false, false ); + } + + { + uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + + DECODE_STEP_LO( false, false ); + DECODE_STEP_HI( false, false ); + DECODE_STEP_END_LO( false, false ); + DECODE_STEP_END_HI( false, false ); + } + } + } + + // Decoding loop with all buffer checks. + { + const uint8_t* inputEarlyEnd; + uint8_t* outputEarlyEnd; + inputEarlyEnd = ( input + inputLength ) - END_PADDING_LITERALS; + outputEarlyEnd = ( output + outputLength ) - END_PADDING_LITERALS; + + while ( outputCursor < outputEarlyEnd && inputCursor < inputEarlyEnd ) + { + // This code is the same as the loop above, see comments there + + // load the control block + __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); + + // split the control block into high and low nibbles + __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); + __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); + + // Note, the carries are set when the nibble is at its highest value, 15, meaning the operation after will + // be an extension of the current match operation. + + // Work out the carry for the low nibbles (which will be used with the high controls to put them into + // match without offset read mode). + __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); + + // The carry for the high nibbles is used with the low controls, but needs one byte from the previous iteration. We save + // the calculated carry to use that byte next iteration. + __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); + __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); + + previousCarryHi = carryHi; + + // Note, unlike the other compressors, we are essentially doing an in register lookup table to implement the logic here. + __m128i streamBytesLo = _mm_shuffle_epi8( bytesInOutLUT, controlLo ); + __m128i streamBytesHi = _mm_shuffle_epi8( bytesInOutLUT, controlHi ); + + // Either use the value from the lookup, or in the case the carry is set, use the control value. + __m128i bytesOutLo = _mm_blendv_epi8( _mm_and_si128( streamBytesLo, nibbleMask ), controlLo, shiftedCarryHi ); + __m128i bytesOutHi = _mm_blendv_epi8( _mm_and_si128( streamBytesHi, nibbleMask ), controlHi, carryLo ); + + // Calculate the number of bytes to read per control. + // We use the value from the lookup table to . + __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_and_si128( _mm_srli_epi32( streamBytesLo, 4 ), nibbleMask ) ); + __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_and_si128( _mm_srli_epi32( streamBytesHi, 4 ), nibbleMask ) ); + + // Here we are testing if the runs will be literals or matches. Note that if the carries are set from the previous operation + // this will essentially be ignored later on. + __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); + __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); + + // I want 128 set bits please. + __m128i allSet = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); + + // Masks to read the offset (or keep the previous one) - set in the case that this is not a literal and the carry is not set + __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), allSet ); + __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), allSet ); + + // Masks whether we are reading literals - set if the carry is not set and these are literals. + __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); + __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); + + inputCursor += CONTROL_BLOCK_SIZE; + + if ( inputCursor > inputEarlyEnd ) + goto BUFFER_END; + + { + uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_HALF_LO( true, true ); + DECODE_STEP_HALF_HI( true, true ); + } + + { + // Now the top halves. + uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); + uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); + + uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); + uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); + + uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); + uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + + DECODE_STEP_LO( true, true ); + DECODE_STEP_HI( true, true ); + DECODE_STEP_END_LO( true, true ); + DECODE_STEP_END_HI( true, true ); + } + } + +BUFFER_END: + + // When we get here, we have either advanced the right amount on both cursors + // or something bad happened, so leave it as is, so we can tell where + // the error happened. + if ( inputCursor == inputEarlyEnd && outputCursor == outputEarlyEnd ) + { + size_t remainingLiterals = ( input + inputLength ) - inputCursor; + + // copy any trailing literals + memcpy( outputCursor, inputCursor, remainingLiterals ); + + outputCursor += remainingLiterals; + } + } + +MATCH_UNDERFLOW: + + return outputCursor - output; +} diff --git a/contrib/lzsse/lzsse8/lzsse8.h b/contrib/lzsse/lzsse8/lzsse8.h new file mode 100644 index 00000000000..dec5297a17c --- /dev/null +++ b/contrib/lzsse/lzsse8/lzsse8.h @@ -0,0 +1,117 @@ +/* +Copyright (c) 2016, Conor Stokes +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef LZSSE8_H__ +#define LZSSE8_H__ + +#pragma once + +/* LZSSE8 - x64/SSE targeted codec for better performance with lower compression ratio data/less optimal compressors. +* Supports minimum 4 byte matches, maximum 15 bytes of match per control word and 8 byte literal runs per control word. +*/ + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* Re-usable parse state object for compression. */ +typedef struct LZSSE8_FastParseState LZSSE8_FastParseState; + +/* Allocate the parse state for compression - returns null on failure */ +LZSSE8_FastParseState* LZSSE8_MakeFastParseState(); + +/* De-allocate the parse state for compression */ +void LZSSE8_FreeFastParseState( LZSSE8_FastParseState* toFree ); + +/* Re-usable parse state object for compression. */ +typedef struct LZSSE8_OptimalParseState LZSSE8_OptimalParseState; + +/* Allocate the parse state for compression - returns null on failure. Note +Buffersize has to be greater or equal to any inputLength used with LZSSE2_CompressOptimalParse */ +LZSSE8_OptimalParseState* LZSSE8_MakeOptimalParseState( size_t bufferSize ); + +/* De-allocate the parse state for compression */ +void LZSSE8_FreeOptimalParseState( LZSSE8_OptimalParseState* toFree ); + + +/* "Optimal" compression routine. +* Will compress data into LZSSE8 format, uses hash BST matching to find matches and run an optimal parse (high relative memory usage). Requires SSE 4.1. +* state : Contains the hash table for matching, passed as a parameter so that allocations can be re-used. +* input : Buffer containing uncompressed data to be compressed. May not be null. +* inputLength : Length of the compressed data in the input buffer - note should be under 2GB. +* output : Buffer that will receive the compressed output. +* outputLength : The length reserved in the buffer for compressed data. This should be at least inputLength. Note, +* The compressed data should never be longer than inputLength, as in this case the data is stored raw. +* level : The compression level to use for this file 1->17, 17 delivers the highest compression, 1 delivers the least. +* Thread Safety - state can not be used on multiple threads with calls running concurrently. Can run multiple threads with separate state +* concurrently. +* +* Returns the size of the compressed data, or 0 in the case of error (e.g. outputLength is less than inputLength). +*/ +size_t LZSSE8_CompressOptimalParse( LZSSE8_OptimalParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength, unsigned int level ); + + +/* "Fast" compression routine. +* Will compress data into LZSSE8 format, uses a simple single entry hash/greedy matching to find matches. Requires SSE 4.1. +* state : Contains the hash table for matching, passed as a parameter so that allocations can be re-used. +* input : Buffer containing uncompressed data to be compressed. May not be null. +* inputLength : Length of the compressed data in the input buffer - note should be under 2GB. +* output : Buffer that will receive the compressed output. +* outputLength : The length reserved in the buffer for compressed data. This should be at least inputLength. Note, +* The compressed data should never be longer than inputLength, as in this case the data is stored raw. +* +* Thread Safety - state can not be used on multiple threads with calls running concurrently. Can run multiple threads with separate state +* concurrently. +* +* Returns the size of the compressed data, or 0 in the case of error (e.g. outputLength is less than inputLength). +*/ +size_t LZSSE8_CompressFast( LZSSE8_FastParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength ); + +/* Decompression routine. +* This routine will decompress data in the LZSSE8 format and currently requires SSE 4.1 and is targeted at x64. +* It will perform poorly on x86 due to hunger for registers. +* input : Buffer containing compressed input block. May not be null. +* inputLength : Length of the compressed data in the input buffer - note, this should be under 2GB +* output : Buffer that will received the de-compressed output. Note, that this needs to be at least outputLength long. +* May not be null. +* outputLength : The length of the compressed output - note, this should be under 2GB +* +* Provided that input and output are valid pointers to buffers of at least their specified size, this routine +* should be memory safe - both match pointer checks and input/output buffer checks exist. +* +* Returns the size of the decompressed data, which will be less than outputLength in the event of an error (number of bytes +* will indicate where in the output stream the error occured). +* +* Note that this data is not hash verified, errors that occur are either from a misformed stream or bad buffer sizes. +* Remember, corrupt data can still be valid to decompress. +*/ +size_t LZSSE8_Decompress( const void* input, size_t inputLength, void* output, size_t outputLength ); + +#ifdef __cplusplus +} +#endif + +#endif /* -- LZSSE8_H__ */ diff --git a/contrib/lzsse/lzsse8/lzsse8_platform.h b/contrib/lzsse/lzsse8/lzsse8_platform.h new file mode 100644 index 00000000000..5ad2e0a8853 --- /dev/null +++ b/contrib/lzsse/lzsse8/lzsse8_platform.h @@ -0,0 +1,73 @@ +/* +Copyright (c) 2016, Brian Marshall +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef LZSSE8_PLATFORM_H__ +#define LZSSE8_PLATFORM_H__ + +#pragma once + +/* + Compiler/Platform detection based on the table from: + https://blogs.msdn.microsoft.com/vcblog/2015/12/04/clang-with-microsoft-codegen-in-vs-2015-update-1/ +*/ + +#ifdef _MSC_VER + +/* + Microsoft Visual Studio Support. + C1xx/C2, Clang/C2 and Clang/LLVM all support the Microsoft header files and _BitScanForward + + Note: if you receive errors with the intrinsics make sure that you have SSE4.1 support enabled. + For example with Clang include "-msse4.1" on the command line +*/ +#include + +#else /* _MSC_VER */ + +#ifdef __GNUC__ + +/* + GCC +*/ + +/* + Note: including just would be sufficient, but including x86intrin is a better match to intrin.h on Visual Studio as + both include all intrinsics for the enabled processor, rather than just SSE4.1. +*/ +#include +/* _BitScanForward is Visual Studio specific. */ +#define _BitScanForward(x, m) *(x) = __builtin_ctz(m) + +#else + +/* +If you hit the error below, then add detection for your compiler/platform to this header file. +*/ +#error Platform not supported + +#endif /* __GNUC__ */ +#endif /* _MSC_VER */ + +#endif /* -- LZSSE8_PLATFORM_H__ */ diff --git a/contrib/lzsse/premake4.exe b/contrib/lzsse/premake4.exe new file mode 100644 index 0000000000000000000000000000000000000000..50f4c86b528f881a6e56a76228e0fcd32ad512c7 GIT binary patch literal 518144 zcmeFaeSDME`S_nUX&WFk0RvR2TC{4lh}EJ}gP0bQ7LbxsNvKHCsVs|3863ig8J|O{ZAj+r6)dG>UeO(A17@s zxcHBgE{T6_RmF;x*L`*66<@Eo@``J(y)Ietm8&aOrmn5{+O-w4=C@XS{kp5Jo-%s$ z$jV&Pzcv?NWPNb%^8Dw$t$$kn7vA67`oZ#F>+e^V*YSPcxFgFC=yFq+|B!F>U#~2G zTjzhc`~&@N>F)(!yD~0q{huI@gdC2G3yK_5pP73}zRW&HVZo>ZkHfJXOnhH3;c)a^ zT;Oo{c<2=D1nuWShohJ``IrB81l>0BNOL-Z8+AdcCzbX0;FlyiX8pRv5z$X>$3D9F zw|_pjqce}3f6E+7=>Ol6qvr@XAOBD7<*H0cUcENS^Lf`W9)U~7Y7cd=(F%v-@+m8? zx*~an!_i(xri}dpzE=nihW-UX$P~MnWBGZccb7OE?{r`{uA7Egic_5Z>Bnq-Aw;EaPa2Pv0 zW`8U@&fPiF(bb(gLDH`~(cy^2eK*Z?$lIeQNGV^UaFOM;>t&C3INEBv+k#7%Y^S}} zXv|#imPTXhkuw|)o(f}TpZW#Q#*Q*`Z^F?U&5rZfJscNPUfOSsPo@2sx@@TR(bB-Z zgKf7j)~!c@A)E4K$N8vl_8op};-SNDB|V9=1h&bgK+CiybOt~{%15ahMap9kc`dK!cZENf5PWtWSNL2k681#6o!rE?g-azdi+tm+g z#r(SsK`g!*SmSpKLdKN?MkG6~LZC^x%{>V>RgF^}nGV;dzle6dlyqfZvWm0g{1jN4 zhiu~k#oP@+)oLyQs;dAcekU0T;QBosG8w&Hmy>!oi5dMwS7p-GccifQkJ=1M`UgYh%ozh*qQ=Z+K7i6B8&Qn`r#HHplX5XhjS=yc&yR7azl9(DnqdREBT7Edx@914XnY03{tqP9*L5g)oN@JlBNG~c7AiIM{TcEt%BBXa!h~kgYmdtjaC@#j zCt z-Lv0y_(kKLf);7BDc7dYZqqJ!2L+Apsk+mjF;3|;S0HMaSUJ>b%2nHKoo;lv94TKW z?2ZTsHGsf$yk{(P<;3c~vixjl*b$GluO}QcTLp&iodT6K7058ukD&pxpYAgA-7N^n z$+Q%sP`V_&)TRClXxrMBwwv2+IXEb_FQEx7wG)&cH&u6U%k4*~5{q|2NYYgyZ!|tk zZa3xuv*mV6t4$e&i=*nyar740?!84sMQc>u56KEGcsJcbf(UT+MrFR=KumQ3JJMB? zOX;N00Gx7-2G|;1-5cTUL;3>BL^i5^N=E!1po*$S9uoyob%%UAqw4Qtq;V;90y=^D z|1z63cGN(FXD7als;wvkGc;iK2M@p9HZj|Fa)0Wk=uFk=7hv~#Z*7HKpvm^dLiULx zQFU`(dsrU~S9Fu|po6e!!w~jOfQ-fO%OUK=3IB?)@uXtI|%(J7ZDS3*aMkUMC{k+GrHSvQQC5!D|tqHOC4ic>5SbU$(XwVrR znZGDF0(+-|t$5WYLF5rKv#A47)x?9KN@PyFM(6#NBoTi5j2)H31lj|+lIK%WsN5ff zqv7Xv_4}#A*m?TVfg9}fBNOmVfk}N{`^z^9F#ww%AZf< z&&TqoPyT!)e?F8y|ByciBAL*R_%r~HC=34O4NZI+FanjaBU0O~B~957Xts7QCt(Oe z^r$D(_2pCO01p0C>>u7CO9cE!s`}#}-M@M<0FsfEU zS~9d`X-r;!uJs=FCDJ3Nvl0%YzbJKe?dzm0!gh-)CxS;L!joeKiDg!UEe@`uYAj+S zk7+_I2z%d;*vO?t)fY&CnXN+mzd-Rz8#m}SrXxKb4O*ws#py@2vXxc_Yd=A4V@HL# zeQ?yjJbT!vJ251GOTw^GW3olH5N=g3BCLS3u+{8WPwQv1&u$r#X%S!r%ne{Uq8jX8 zc^MqeW-TM80p}z=jjie|k!U#$%M=6`s}pn3KvGn#OpUM`j94!{IYgi|fAM*m6z#q) z;zcVk}xa&E1PyF%eUN7Bs6-#i>Wc4pP%KO)feo1T&FlvuXP>r z-r7ZR<49p@MCTJ<`#;3&Q_n)=&mrf{dG0L*)6Ys)OusByKD{zoHvObjap&~~mC4dg-AQ@6Ac^K) zvK@mYO$uSr<`d*=)9}+eS`XCL^7DXFYFz9Oz4q?Ci*%m2~YaJ}YcDg%qt4 z&TexaHZ^=R13wVRytgvutX&yV)e=*WGTfty?&nj6SimiV{}l;CEP#23F$3}n7=|AD zF1UJdet%nA;5g@Xy9d;Z`PsA|2&BoWopqOaA0Cr9tKB>6&k$Y8b-`bf&dn1V?EwXL zB@4G4Ti;pdNd4Jzch;SiD&3hUl-Yq@YyOcawDJAeEf83%xieywBnpk4g)sQ9G$(nG zIRA)d9}m; z8xJC0eiXEVVCf{9-{Q^9)8>qzDWaVJ*8JN?hRMet&@A@DF!`97Jcz><+jOjbJ!XDU zcf5nOcCu!xd}V8}11oH*PsLgjS$dCaaAd6!Zp^rzkYaUZsaN8QHnZUQaIdJM4?#|A zqJY8yUAUaW$BI?B67BHjlKxM;lSAmPoS{dvQ~?K(col7^22`7E&{%IG4RXN6YO}Gp zdwdYAUq_?y6*r6Bv=gP1J$K(LjXdMXbi_B2tS*3z;G3V3kSGwl^kLk6#)Qc$5s4uZy*!pqD^9wi&WC zK8w;%$?(Nk#{U(+m;~DTt2_&CNdGg>d1yTX+?RmI43E?8094YIa9Dn7%?5%YL|MlQ zRL>qWh@umTH%q@qO~X;+yh5N#0j<@t)N0W8{6Eew-(Ba6gavG55d<@dtI@*?Q80 z@yB^%)IXJ+JvwKC)bfFLl5dyfNbgEtBM82@l zK9kiDYl+lIslJu!mRkhCJwV^qrki>Xrf2Se%L3k`rQYjRgB+_(HaG%k_R)-d@8!D@ zAyl{C)3MSF`ApYVEf7=Apf$Slvoll2OlvAgl^WfJG`tYD4ndwN_&%bpt?kJcAVP=g+UgeO+Aa`Dd>K=PfM~zQ zFQFLzJG6zdLnPM;QFRE+iyeGSG=3SmX)SToU0N?=aLT~M(7xa_dxBH}=73l-s8@9_ z(%S5cs@rZ98t59xW?AHz{n^^6Iv!Bsab!bIZmzen+LcXt#k&wyr%+n!pr{J{nHe*C z;djjz;)OUA&ABulLT&S0NF8&p`YCT>3ipG%I!upqTh)2MFD~3EGLE)D-?~ZFeKF^! zQT0bKb9nE>?w~f94&krY_GVCgJNd56g>K^pcd>19nPWR|D0U<(&3PW&e(Mf+A8sfK z?Aq`yw9@mA%jj`CpGmv73Z|EAhe$m!^9G-Bq+s3gfi;yI%FJ26^Wny!aN2D)mYcJF zGhlT4?9TMC%#bI$!j-KNG4}l>niX>CZnMcBXsSvTnZc$d+s%5H=`e#Hohkiv9qO0} zpj+486n6By=Q8JbjGjK{^MN^TZ$ppOv9%A$9O&y9jcrM`^BZn+rxlNB;T1TXjBPyf z5e1U9fj(%F(FOh0N6ethbep|B`&>r1+xu`!iE~%rbKQZrxyL%g4El`j%i)rF?o9n6 zbGF}{?J`H0vps2dZ_v#elC?jL8Qo1eNe0SP{Cjf^>t+taqmy+(y{#v!xvs zC}u`{X2YWNoJIE7?Rv=f%yi7VX4r2wy8}(`mCkgNJJ;Ae z&!a(>GmuKQG4RZW2I-y3VNXxL%h?mywxOnLcgGYnTwWb^x0xhW#$M>8*BTEQzx3izRtD9j?{SZwQ7~Q!q=jDmq}&+ z(3(OhaPfS4kcEaUN}acm7WoF^oE1*a@Ja7hG(V2LF>9! z)pjkFtz~A%te%fXTJ3yqd_Rld!fG|=`ON67o_9tDo>?=(+*aKqHJs^=U|lr025$Y{ zb?l>cjB~UlN1EYaAUtceWu3qjTF06l5#gP1#40sAF42#dSWe>wJu3)?n@yh;)quz9{Za57QJ=%V=DtrcXU~GFq0$m|FcZ z=vw1Whx#(OkfO2PP`}Kv`fQ@mMwLBXIfwTkl*ijpM%?Bz#{QzDZ}#k@#|-<-TW-IT zx>gYYHM!ZB5KrzgfIrB5$T7Gon4{JFpr8K&@!RZ1HngB$n z9pw9-T(>+73wpevlphScSx^_Sn#4)l!444PDEbJZW~aY<6lv&lU}Q;ZU_m<6 zU)@-NN47r_f9}*EPRbv}G{nY{Y&*JQzTb?w11+K}THH%>^N;qd?XC-Xyx-|I=h*nf z=yDhhZeL#BQOHh85_Rof8Z!zq=H0?u~vU>Z!&FfbM zj!ejBPionZWvHG(o)*`+NBrh6H*Fz-9c&*ujQZTT7&o##KV&S)VMG2moip$?x)RdQ z>6{-!esuPq6jn~>AWnz6n&!Y)JzQ;In&pg5l{cAlaRjocj3sKE{vT+Pr zZKVY44Q`Dip?!9{=lEp8>RMKK@38gOAPv8Kh-r$tA<6$BsdB_C2EyPz(%$6cmv1Y^A=hAfBjjBM{E%)57$i z7N-3}yror_+wL@5qvr1)R_EX3uU!pO8DOUmsc~3~fpAr7bbJvaII6;pG9W$4zdQ(r_O!K0u`|V})JAX^o!^(PNAucf zS)519fE+-bN0Sh-bphfAeLb9-)Kb(9(N!%uV6tq!8gkS~}qzkbHSR%xG5 zi1OuZ7(}gDG-{bYZ^|%ndAS;LFEg8_B4d`No2CwlQjP8zbbufS#J1Yqbxr+ACkQfc zVA)})qU2OI8O7R3-?CL(JG0C}XtQf~GN5sIu34zDccx%(@s%2TRz<$d1YKrKuFMFE z98VGGmeI1f`&Y~8!B3dGs5>*KuO%~QpPr9nWH4ov-tRdE&?$WW+GNW#$c_sMyN)MbvjbruV(#9C;mA zc_Z_&kWe2uwG^u=tXE3db`2~5-0jBoZr8dK%n^Egk?b;(yW8kVxnK7@bo z!@*(=0yr0;wTs@6gS%+SksOT)!~!Q(V&JhxMh!3cLtXG_8w>3%s{~!lI8dHzGL zZLD{(vF3w5ZDWl^=8MV+9egu%@bx7FVl3t&k!7v>g^xMYZ|BlVs9In?s zvys_Vz96dWZQcRhgKftkM`;z_ayU5xiSLJK@F5E+9hH~c) zYtSb)z23?(iP|WGXlQPzXoW6%p%|B`B4byU{Gd%saFfxWO_gk2fdC`f)pg)jd7!TB zA&a03e43nr{`Otd+gBj=N5@&U-C}lNSV!H?J(+qR@m4SkD;$?z;ka4h=moovV)Ijf z&Wqlb7QIPSdnjDD#?^5-jE|=-0_mD>sga_=QdshdBUD;M2nBrhPc5R&`7RhcaoskGX;duKocbcOI_<;#IO4sMp=8PEnNL+!5UZF;z7ZojgOLQB!MS+ zTr`qdaMm8n{z8o!hf_+gY|TC)_3w(eOT0Ut_=>NK zx$Rbe$Z1a=@5PQJk||NV$R-oI^cWX85R)OFahjvoAr|1)8-Z$}L-zx6Oc|L7T*X|DNn%I5v&Kz`NU`1SAFUc~E*_!7|k1;NH5k+Bh z5eBX>rbpEwKWUn0zr3EI{dT&H%p^ACS^3$`eA7d>^XrONmPFOLvSu+$w>UQ!*y-8g znK;qjX|XDb zQAH#aWN*C|TNwU+k&*GuS}%V@_n^~SmK;|foMUf`-0V)3 z&Q6V@^4;+V$hCTm^>;hA$`UK)4TbKGw8Wp2`svW!>Q3FPAIGH*7uf{==Vc|fJ1(m$ zTn>04Q}0O^XwZSylli8XV6zM5&l)X6pMh+{ZBD4nhjHRo4OrHo4J=I5@11agY;Q>3 zdHBTbEgc&d3{jHD>czb<@-gwLnmTpz3t*F$?5=Y~Zv)w~p2BG~DVJ2Hg(d^{C7ijW zt8h?LGasf29KKx6t-8LzcZl+?TC3W1rKON^G|#p=}?0pJiEt4q}Bg&u4i_ zn4`(klg4+y%MAHzvvt#w$Ftxhj3%c_&Lp`J_Q|-m8f@DCjfUbL5EoVNLUa+@_%;-W z($=!e?6nRL2~*1wvINp&q*g^v7~BGb=W(4w$~wl5lQ`r{7BAU9mcis{z%8W7Zik=} zkq0|Y*9jMYXt$g^Q(6{JE4vrWc;bj`j=X@014Ph8sRVaCsH1WKZ48BV^_Rf|APnvf z*d%j^)ZCg*@Z3%-+QF?hG7d>ubU@mz(7@T7xw^Fx6ojjg$WvnKS3=r)7a=t=rfSB3 z#!PvMxn0M!foC-Vw8?(PGHD)QJGDR*um5yOE&*7XiQV;`fhokaYF@og1K?x1`x+D| z#l9o03`ErfNLKcb*g>YLb7hT=Bc&*HoRpb{`AQ?18w=XnyqlIX1`%|EksnhFYh=*1 zKWOS7WQ5v+r~_3|^_N_4k4kTJ@F&*Y8nG$e%wn3M$x+hTvkV8_(AH+N+r|Usu>6qp zVMx%UPuNBH>%$w3|+>8=BOW%M>mYn1Q^U8+w?RT4?Sq**oQ z3)RoJ5V*1ks>nVhEo8c+L_`@vCl5&r&cjgVC-#O4fpt0``MiT_?G(ya$hg?aBC6pb zfkwT-co0#Hu=bCTvwCy%pqOfDL7|`{LUU< zAS1GgB@|?jv@Nr?pzC#S*KM#vqBy=u^VH|;FEU2;IZhdSTPmWOac45AC#TNt?GhY` zf@D`;0Is#OvdJQ&e`M+d`QyoDmDyPny8mt$;o6cI*3nyuW+WkO(vtsdk z$grlg#&6Nk6ar2Bc3pNF?f_Y`>a8+@I(6iEEPjVB_mOQM?Ef)6MAZM#EbwNNqmBNF zNw=x1SnubCZG%UQ6M9{OXkY}Sb|j;R7KzUVqikq`Cc7XpF*48>@E+o@wW$MxeQa)& z$B}Mol4*-Y;rH>X7dm8zS;H06aQUs#>P!gP)J;M5`5?6;qXwrJKIXsL$b`3v(u!bhT%s)C}S ziOG!jFkW@`V~Ay~X_BMkvu6ngu6$2(=+1V%jM2h|XaaH9hyqU>S>)X$*sk4;+iTuH zoj2*p1l{!_p%3$(D2UGj13AIijN6v}VTfHSD-j8!HPn>|iey+o{x)C{3r?Bbu3$Me zYypbM^u#aL*!&ZG18q{j%pQ?k+a`w<6+-zEv1JPYq+=Xu)NQlBs~F~ini#vBoecRqc+fwnVxanY2e*ZXKJ^%!`l zULp|-Z_xK(NXL-YISa8TGHgQ;oVsK?1CZG~8PU~d2O;EAO7i7S2Bm&f%Q(@R$x%o? zwHDIQ?j!u+As;54Z5~QP6SNGpO@$VglDhTBm~`EzSCqF;mR|j6V{3FSf2{63zXrqp zW9MJGhh#A(N6KR+oO2+Gp%zuwl9pZV%C061`|ld@yRH+-bZBaL=scZj ztbf4KF(I3JAQO5#5_jpWXFt&JSU%`m_SvH9WIBa(WZ#-#;cvPJ`A#H&K=l<(MxIFg zH?*Jz#>4Gw5UW=)HmNtbLWVrtY6aAqAHhhF1o`O^Rxz9(k3*(pxY#}Zk&p1m?|Riy(@swK}cU) zn`+G^!aV99s3*<;T7>%=17CF9sm{sQk_C-=h9Vq#%10r^AE2qIT8EUcE>pFRi#n@F3hIF{9Ee3&y%oe=_KO*xh2egP_9u zFtWW1joQ^sW?RaV7|se@-P^^}-D92D((;LSk?YOQRbg>GxGML5oym!+s}Kcpv?7@I zjtG6n?C6(9tZ{%O2G)m25pdtrw2n2DJc0G|aZJ$VAVPBP6_E?f2j7w5mqpbxf2HBv zN=LSvF%_}zVH^E$#=AKzrC?~cdJD;+>Z0js+AWb|rP&EHB)0Zf+fm*C_ zXN!g`JAv_m9NzAM1VNH+O<P1u0%k2EO}@MTSw*Fz+WgiDpm+e>8X zx6$#t^>`l<5e9U@ybsbIodKe38~S?vWT&1*S8%Xf^$S;`4=e343En07Z;+*3Yvy46 zk2If3?wQmcR!L?~Pndc@SBvV^utYl*OJ%+v)K$@`RK)clKx!1iAgtot>RbDA;LL!t z>@}m#$Hk7R8t;R31jzHclGsFJG`84HEHcJ9-2 zl&3m2O9vu$;Zo@nyTy(IQum_b=y*V{#Gip2G!(b#mFd($s?btNnoZip-RjwV(W`V( zdv%pN4;uHc6hReJVM-^8nEpV#wL*?Rl@9rZo8IS0W<#K0$Mc zgr%MsiI;0gqv~$tN+e#x8_O)M)ygeYyqCsuOCy|nS^|oYQT<46TNVbW2(=;$uZ3Q9 z%;$!YQsh`hM$jIFyp)cS#H^J2JH#J$NvpZH`t6zNCB$8I)L{|b0N}!w$;k*m8EDjrr{)sBSoe{+q~F|bjgC*%&>jSowqeddmMz4dw5m(qhAR%4{o!b< zdRY${N2n_`35nCHex?mQPv^9gi4~fkZE01XiJ0O*t#vlpzeMENmhJT}=V7ZTaM(Nl zFu_@zvWW=73TPfWMRpSWG_)KQmInS1wIv2x)b;Z+Jp}fs#mN8K-Dtl%dhl95%$pST z03_kJ6hfNV?5(E@X_%!0k&3?*bDW~@PzaQ3iCKD7V_(ZPHUktz)n&S|=|hcaC4)Ug zd-1D)9_6$^1CJ}r+x0C}E-Akib=PJnM{-qDvYFkKb7Rq_kL0bag+vmO6h}!5{CeXTZWax`Ypl z%Ng&~&Ha|RYEt7>-H=vW zTHe^S3mP{JHRa1S#pyY_p{P3X&1|+wMtdSQt3IaCmA$@9l#~lj5EXJMC1i=f;k4sq zS`ILSINb4*MAZyCFVk2s$jHJnQFS$QQv15uKNDha)Z!dkBpQX+h1Ais?~VHrnxuSg z6Dv<&We^fQ9nn?%<{(SU2!DW?!h%UHu-EdNgxt|dxg(rXVgYQWrP=)H!(CTybwIcp zcH=-UQ7Q6*a>?lyy<%{yb7@YH?2{r@ykJu&j>t=)t(|kKM#hIL93WSLG0K~JW9sL7 zHI9%lqvMb3d7TfzL0(@JB#+BZ{eetU{0*HErD-M{3|d!Fn|7{*RQNi8OqT>TaDf0X zrWLVd1<6q$J*H+*$m)%$>t2U2_Gac5KuuG1uUIqm+OHdbO}mAVrmB?kLRW(r)JxOH zbQC&>J^~a^O$PlEJwAz^<29lji7pr2ToSJ#E2{jIichmAX#moVuLt+>Ij#Jer3*ZE zM0#n8KR%nJ8JF98(MECEIl3mKXcYC4v_5{27alK3JXCFj32Z_Hwb zZ=!>!x(_;uUrr_vZqRwQ>yicJzpjBibtwsf>9*>LixrJ6Z0%tSIN`L8W2UXsWkDq1 zm}lhHNFAa$iL6m4{3V+`yti#)x3TYt(e2DFeatW5hTKjZGalcA@g%ZomsY`e?}6MN z;Oaly1H3|u>nbLLO&p(5;jn6D+)vWHruH&X3>ip|IhMyVO;vu&8$X~8vlHM*G0bc+ zcGL*<)Nv36M0}y`5n-iL-lhRXTigClB%1a(v?gWM|9!nG@>ncR{|p}?7T7Q|MsF-0*I8e+d3Pyz({auV zJ|9AySY=`q2sex67t5?DP`^g1LZ3Ins)Q)V)J1|xwD=_5c8x?F8~X}#;zWnN*r98} zi9J-ZdGcwS^tJH=LD&Z1KS+$J+n_M99zwjy#(Q``RGWB74qaa(JypM!o05sr|Ib9R zvo*6y2o+#2d__pQET)cOV%r9^4*wViMYScIc7(2N z*~sL@)SsArBC})a1|Z=5n_6V!{`PLVnJFZt9N4iMwx>>uQfv~S2*52gPUb2|Np~Gn z^;+&$TC+*sFK~)*tITCSB(PBJG@55*pkgr{YXmUKw;(8_Z`-=y>1VpW41GO3I{X}Z ze#`?9#Ne2BFg|@Zxk*2yVay01a&G5FPU;{N9wdM(Qd% z#S0!)Pw|D`TfprSLUXu&p4a;%{&b_ew`3HUYR^aR*}EUO&8YTfiL}KfEi4Ojk9!D~ zwA;LfFTF!sLB#B6z??v=!1j9cFXo;Ft>QhyoE9;IYgsyB0qe*Qn~~|g0x0%%;4q8z z*2abtFh19hX-XD1B}X--T6&AIR~**t-eNWb9acR)@MNvg{ZCZfnyFO19dz;aH*y)& zYPn?Qn3(z}Av*bgNOH-{Ar5p)}~?J{pwo@9XV7!V0uH(fb`l!Dz&~gdqt?7JRg*5kqNDNm@wa^axDrKFGHY z)|k=A%XWFIdaRW!5__!W`Q^MU=5HjeeV6sl zQEBaxrdObzN|nGv|HeG1;f=5NJufE%%OjEg+U za&4t!{=LF9nBhuoGeSi|5xJdwdmSRSQLB0htf6=b7Ax^?`X$A@x9Y2EE^>Nry-8|a z-yFy z-4{6=XS=>-UBNUox^rvxR(zI#N7xX$rB%JF_nphF3u%bNXoU0JI?2ZwjTE-BaS^wk z5DrQ4!dA6e=lKbsmj+tZ1B?$2POpLBJphy^c>I5|dN0wekj%e9tTr?6wTjS0k z&stSRH%ADrzpl~e-Sk6R$gc4j^9HiUy#8#fD&$Cy2=0_OyT;$DKBWsS$LC2i#l}1j z9;H_GF)W3Y@R&2pw7~(O>6zu~4YM=*%<>qzcV#xw0HB(ok$irM0y^ zD#T^FPk!d`YZG_Z_qp=P7xF=NJ^LqMm9`{~V_hubPr0M<13gsBjVBw>RjUQG_NSH|Xp0C5Q^0#hBv41nkuW6W?SirP9VjTx@w zF?FQ`;$ZJlP1}1cbO$gs? zD^2;4yWzatWxa{8v&sphTi9>RD1^3iXL%(0HKsg=G!8Y;!sw%gzpXB@+=rn39~s#U z*qj)hAe`>R2o;CL z$uO1HSlWu2uG`4Do(Pb_wt;edfqFmZpsxYtR*~}4L{1ioxQUd>Q6lz2J?UBB8Wmhf)_4}`=?sj%DPP#Tdt>|Ht$VUbLjGZN}D zO%|NtkdO>m&S>^$^P?L=(;7;F;&yg4&sS(cEdi)uW0}5nkxA zCIosmJeiqUz=_sJSTkfAvK*7#-qD?@&u+wVzX5B}?cCP0Z^EdZ=~;gmuzqSAmT*bP zBTOK|MeM{r;==6!SIo^oQ<>nx6H(kJ0(mdl-lC}+R!{D!ATdFRQ`QC8m!Ax#mRa12 zsZ$HAv4i<4|CBgZB=Apk_cr1D@hd^cU-=y1bBK=vG>_!d%5w$Zem-aLIgd|}&m2Bu z$eYeb6NqR_Z1ucV*c(zzvJMIF6DQVrP07lp)a5z$c#dhxgcPLYO8})?-K;k*%B+iW z1+E1&IKPHh710}~!?`O%b)dFSTOs9n%eDe56dQD9033n%o& zp_ehcNDQ5aHH;MuyW&kaoj&*~N8p=lD#fdCAIuLtF||Z*!>z)kA4LJ`>(o#FDGg0e zuX6XweJg`THq6=@Z4!X)N}SVn_%p1|+TEE&1sDS;X9Dgvet~RoZ1A;4Vu_OoPtn8m zTjkscrV}0|!3>K@09lRIxfSB5PKPUV28M`jW5(#zBoXq>tT3g2+p=_R&T~FG%R1GV zQEHdsJSVCETL0E5LdIC|jsi>> zP?~Cj=cBNQAWxs*o*-LiCBiSR-{~I2Wd0o~G(@+Jl}E3BRHN2Zd6SxEJZ6a;fw@wq zvWZh7$q7xVCJ-G}Z-Ih{m~3m@i4j({PP_^x49931j?Dsuyc;uyo4 z@>%R4m+ADgC?$P=YJ8DR^)9w2jT|jnwv(yPW{&4j@jDlFel}4DjMXzw8$T08z?t4{ zJ!%4LjWMz=cxK}}vR0^lCSA%A>}?zWD$~ovJ(PtsraF4mm{Hr(#lV~EMn+n$MY@LN zCIHqCCu^Ym_8{N33xhq|NhVvl9Gyq|vN>q>%}l5IV6ndEWgP5kZa9Xwwa>|h3;v`f z2iaz{LWF*B8Ij!R5mTdImO+t{C=T*^=AU?-H_&SIIJp{X`Cr6M%8K!^x4%J%+YIb} zGljc!ayoP{rbOjqMD*>XN@zf8@92dvxE15cU3?#5hx}PVI_C00@hu-CIM+9{yj0$ z?@=yGNs-2x5Y{{$MWfZFau+Y6FKJ3|Mr3?ABEm@M8otLWfL9$oh1@}MLTJbVxU zenJ~D)j_wsSWj!DKrpp`#FGg}shj@>fcZ#KF<6JjA>5^odE5!zQo^`>Wt)|{Tt)Fb#4 zndq84y+hLPHvL#V3K=7;?{Ghi7bW8C3mpD zxLxk@y1pk5`*_}702JmB68QaYoo@rq(6vs^C3s1)qWBu)pWGpl9 z&X&x~k&yPax`cRaopY`85=HS!-9~Ay4ZF~6yBTZjOlj9nJ9n_+2gcR6JN7@qnAzl- ztgFAkL`l(j4X-hEidTZyiNTwya~t(ER_gG)F@)#Er|IMxolJDsJ#vE?a`x`{be%DQ z3~QDdx^IhP`qy#iM|5fdsp$Q)rq4`HoSsZh$fh3DeO#q;|F+FWD|@){%XLb?ZXT+7 zDx0ki-2r`t9@m)5A?r$@gotF)_n>09(CEp0!E(ExCIh+Kh8ql*>>H#69m*==_ih1c zFbc}vF*qKPWj{@&IZeGOy(O!FaBxdjm`nEBywHVqh>eo~*rA}VXAyP4CPi)%_GgK= z9ad%xEt;-yujB)V^xn)#PaFja8-D`dX)jP!KV7IBk+k~|0($2a#7el7qsFi|FI77S=#on%VQd;8`bP-b{D8}*#F(!gg$I4WK zN}ALw&JmiQChIxn?o7-ZyMxj3Al0*}JNC;OG=UE^rXGBhrdnIoci)jxqBiGbQ}@Rs zx}OxI(*(6hzjFUEwI7UMrQcq@M&7o@SLnB=_1k^%PW?8Wl2~_%<*n*M8rIXP$$YcU zzLUZ-m8n*B!m#Y$#P87A*XOdYZB3P;F5&;@x}JwqOAt{J++zJBXuN^@qI%~|!| zqiBmDPdWKop(k`2vrH`ut`)5}4e1EI%1-u30xwAwh8u);=Ntbgc!%hABLo25%1IbH z4GU5JNb?;I>~{LM3C4z(y!knJxi9DrW2cjp*_7t1A9ly(&h`tem>Wz?MahhafNfPj zpfSc&qk%MbOdA68#33+k;ZWg_xz_w<2mbw?UNJ`DMp+yoH|%9cUQqN&HbG~iLL#6~ zB*I6Kvu0&&ztxFl%uQ}lqQ^_rbn@`>Y_f%*7#y(Q_k=lfVJ%MvZG8yRkh5``|zP1A#mv<+L8MJl_bNUBkFZf4=XEg zx<@%fsx84|X-7ESjo&TAu}&@tuThVof{^bc@NI~@V^YrBC({_$FsWu**RnUJ9-@O- zOikDQ;q~EFBt*AXja3pmFkLKevzX|iTVz1$zbR^T_(A$QoRT=8-bBW?aX+=1PmTn3 zv5w0;BmjGIn>}$KHTBJgGA+lvXvgAs?4X)x;!09$52@L33@4!MTOSVL-x2x3~YV(}ZuYK7|SbDQ+$h%vxxyFXKF4ChTJ2!~Tm~ ztQ(Jxfd}?#&hDY#JERt~$0iExC8=C7oo>^Uu^Ub*%s$ z+=iYDi)OVf?t%tDscA}8K_|5@K`DoO#BNQF!s&8^OU|0AtYW=4u*-Y*Hco>di-y7V z8)BCwI9GOAKC+4h|H=1rk^Hi%my3+rJ2!AG)*>+7%BlkD{T{wVZrp+%#B0{#PcQV; zwbgV~W~Y81UOMf5au>2*ucyGwv|Hbr;R`IPp{sRo;GM*KwRrG4CTk?^k|;L@*cXl& z{XRRS@?8uon1PFQhWSA?o5<=8$I`B@-Pg$FmAmUGP!A@Da!NalmCb_CWU-4Afv}8MNY9XT!E8tANP~_wiJdG~4 z+UVdo_6L8=6fn71X9MbRxzIITr%ny5@^cm4OgCE!T%T6Vl~Ej+T*sce&(Tq`bsYwn zhAh8865!^!_0@T0#_aN}F}ot$TkpGta=rDQJEZtIW&`7@;K+Wu9w>R9j@yriNADrq zxW1f$2h;TxTkqu68`_4mHZeW>4PU0dGNN8;(^6cIM)Q5xrEe`LwQq)ErxDKIz>PWw zbn=2+@*Lwsd%gTPlmn)l^$m7DF0nIR8#$Z$&PB#&<;ju8XUB7>?XyWK@#yA#CU%$v zp;p4`!=?dJ3EG|KuBDPI`w-?iYHP{4Zn>^RuC7XGZ==tgrit$|a|1Ul!A6sTCz5F{ zU>;b$ZkVX?o_30<`ZM8EW`kt$`E!8QgChaPVA?(
t@@HVc(0Gm@Y1VqZuM# z>kT2n)n8j5pjadw#$8((=?#1C<5eBecQ$#9Wo0={z+iH`emsP2vpWz4-Nw3IZWrvO zVOz3YgP=))b-B2uML2fv0a^1{TE?r^NTjO0DU!P@)<5K+wl@e@Oy=igv{~noV6DS@ zb#wHZCyI6a>dOfo5XDI;&nXT^C*gJ6A~`tAXf>wS#r;^^I+k)Gx#Hz?3nXs!HL%O1 zp=8xkrT)LDoVsh9BoWN~kj3 z$JAI&B^B*4^-JMlu?yuY(qF$sSGHe6IIi|3yo=BpBMUScWdvmSYNdqlAgh>Ct>$Yqr1Hm9}3765}BBgEP#a6;|)lgxoIBllTy+efNs(ptAQB0Vj4eb~R~oE3P$ zZ1Rw$#LZOJwu$`E!v{2qC=9U~Y@3BV;-b4|#pSs1-LmcBTv5fz47GI?gNE<^x_QDK^1rtk@Ej6QlYkVMDwR#-)j?B)ro@Y+V0IQ(`>sB!jzUylq*tRH@#*< zUwYPSeP+W!zPMScpRaxzkUNC(=rImdf){aOMka4rJ2$g3n@%0<%zo~@`2rYy%c42V z^lb0L&+@Z9&RyPzdjq??w^fiM{<;x(^1O2Ygs5Hb!!x-S<&CcOQtE)&pANli zynj(%U!_CJ$e!xWh}GwPxOHxNVP9aIHzQ`B_u=nJxQMwey>Oqg&sn|GKF{0a-PB2L zleg=2zRb|OX5W^ZrC`s8rIj@s33V)ROt0{EJwa|JTHt(f?c6}?{JRJ4@bH}h`S zOC4vMGd-D0amT)!DZkTn3gjoHe21OlMow`~*m4D)S@%8|NY8uMoCkj2y}uxN&7is( z)^TMGT~Dt`H3wEzt$)jE)LHx<(|>SF=co?0D~OGDY4^=Kzn9E%Dy=ZC;Zb=Fd+y%uiO$_ioz3`~2j!GEC+% zb+DeP3ov!JgpeGj>yiB>(z?zCgbi=jrFm2@ZLOy)NJJwb2kQ| zk^=;XcyF0OJ-~Es)2-Ar&F4TZoX4c$^84OQ|d)8N!aMpI#&A zXM_E@o$omBk8}hbckx}s6l2U82RyR$>DGeO`|!%y=@imNN8|+F;&|8d)z3O#OE2+Z zo9HXjxW;&P>XSnab6nm{5}+aX7=Kz#9pd%|uT8#Z0$PZrY)qYSmo_qJk+W5YI>PJ- zUGs8Zny`m8Zs#ZUE%+Sg9aR7OeHto7W)lx#Z1c$lhAw@9pzycguCbD^>tR=90=>D1OQEY@=BgCnIK|i|7qH3A zS>&J$xH@YnK{1ne6G6gZP1qmqq~Oe0<8{NcP6sc z9pd&<6@`o@EVhwcIjHZmO`T#LkGeGH_=&$rH~6{u?ifyraUsE{B3^cQ4uz)`rrtEe z70&$0D%74{{$M1w#H$JN=yTRr0G-8(%02kbJ#u4Fln2a~Dy#ZmG71Mv6`BpM!IuV? zv8^o7;$OXo2(-b}`kZ?~Z{c~e=c~_lG72p5y=`XHXU>@wn8S_OQT6mVrk3D}Orz@$ zvuogrIP=y7YzKgQoB%kd6to1U5VO)<#+JGtb6G6&L#(^xc6(`N>n=K6mJLr` zRu`VS(wl0EKc}mHs`q&Hx|xM_kjZ7fSUBrLl&}6ydLsU7tNdi~vV!o`JQr}Vg$Q0|VLXWsGp(@(C-3ac?!QgO7A}8E z2(mithKDt^+HtKz0Qvp}AWfS1L-1e}$Q*IIf3H9UB=tV&HaXfpiu+$wq&KKLs>N|O z;>-ErWFJC)7J&0K`d0|{0vDTD{CIF~xx^*k%#(3-pF#lD;R zKqPJ$LDW$ZEr{d4Uv4byi>da2&|;;zbMPiazoBU0B-u(oefDVUQ zzakxCOZ35Lx_(6_v{inE_szqkr|y@CYoLN023a1FdbeY#*uvo%1zrTWpk*e zFL8*P?Nw|IT=?^!|9s?{ao6;gMXm|gl*vxdQx^z$J~>Bzh)2+p?jQ=gg8g3I7ADOeeR*S47k8lhYmLY6Qe63(a9GRb1F2L5%*wbs=tW@0gr$Ee>o?ma=KV z)g=?+_aUG8#jP>%+j)qoeXC?Ro^)}#-Oa=j$W*k3@9LX_^;w??897BjKqMt;8I{na)`q&aKo?eig zxJeFF>N|^_NBuDia5i<#>E)%8_TR8#xmBkjRJ@A?t37S$%~);4raN@qtg z7g+l_@fE_B!exR`dmJ`gEf&KRxu5O8D_V`^YdiSa z*VaVJt-f-;Jm=#d_u5)_Zm0i2K2@c+FNnw=OW3XLZJBUN)*Pt`86WC<{4g-b{#d+7ZR(o#0n{uS8)p`F1eA*;u8IILeK45~o$S#NBbV z`K(^qqs86M9%8WE1S_ilr3oP@xQz(DYIbx+H>IXTvuCRhp?m8jB8m$LV2UQ(5s4>0 zs9Nkk*#f2v#~^j>0lb1-Fg7=Juh}>?s=L06>c)CEv({0@iAxUuRAi^#sVt#4wrK)O z-0RekLLKIItGm&I1F07q6s;bvVGhjIetsLZi3Xvq)}DRxyHU>w7p2%C@gH4&8s#a> zMS1irzd$w9d;uYfghL~@W~UY??p{JTlriMiY_hOe(LGL zF2v9XSAsga*eWBj2Z$Ofr|qrkQPLVYPPyK_?m%7^>ZO^Gtl3z>ni)^@jqG``Ek_$$ zDBqf0GhJO{KLpiRc;FmD6PcqJB9F6qbl8t`c>E`PN9tT2KM?t{X0o|~nOCP%(by4` z6KjP8sp?pI5Jb+~6yxi=w1HzX4;F+(SbG9#xqX5y9wFQ&DGe=*}RHRZkWOhVTrIn@EKPhaNv=`a>N(LzJSne zB)#yB^t^rQl^;s*#~TDC^d$%@`z!iiUgbFbuVv2VBF1w33KA36`Z!Z%t4oo4Mn@cb z+lL*P{HhQXp8R;-QuoS5IO)T2Ry0LeKYrsK=ixoE=%!ul&98?RtSOue%EWxKc=U07 zdn1;RRzLq@7^Y)fvWkaY_p*>U^VB^{bD|n@!LKcG$(e7&W9Id~y75V`SzNcsz0#eW zWA@egQXeNO7A$B{qiTV84+3gugKOlL`Tov#oasy5o97hC{or#cDj62hAV0N8u!7l$t}m2;U9QET3~a0!jin-Yfa`=R64+hl zkfaLfgqmCl7AI3lxTbL$_x}c4DZ({eC+Q-UxgzCJHHL)N#Kp+i5}vr6G*5^ar_szx zUmd7oq3Sn_Wg1FNMqkfaB+w?i9WS4);kJoN(A_O7-EU088E}%>QfBTF&%%07-DP83 z8^#1K8{=N@oS*at;o!i|b^EmkbIJB=y4g+SM^$`QALx?#GPg%3a(G;y))cguMHR-?ffp~3hJ4Gs+0Md|h`b7#8Umu{bmEia~5 zdri8%GTq*kZYSTLZl9KJFH5&iPqzot?G?F(qlt<1kV<{aYG~pySH^k-Ij} zFT!x^+U?yWlI>7^vCG@l$rGE$+28EoAyL?(rhQk2gW)Xev8zQQtPWxVw{E$#Zf8Rs z0q?V%yVFfUeh5W^hzXIWL)M-_eL1>3==;(6af!A3Bukmz^J;Kcu)LPNwFT+`*CEj; zJ<#apOY71rfP6iHI5AL; z3;cnXQ?GHmu1sfmGX4;BK>K9*9fvl4?K`SA>2h4d3qlEB*Tt$Zz9?0tmh#H__hDK8 zMr%-5T#)fnvcep@CG4+sF4!^$1ph zt((mS9jXFK!JyAvS)QyyXH=R;SR0s^dW?^J&bOQSfnp_#BE75SGOlzxxcB;gFtF&BM$B83rgnpD;)W!Ls zUtYNi#qcp?%TMigmqjGaIz#7tn_||h`tih@DD)iC2qLZbWwGZPJVy7J2y*`9Gd1?? zeMO83eko$TedT;Xs<`v=q5qqW-6RVv?1BLzMvWR24Qe#0CSAZl$R^yvhDer(5bJeaZHurA zs38QG*lZrx+E!cJ+MBKZdau1`u~kb%6BI9qR#9xF)#^!?ws@&Tq3rved7e#z_5E8u zAF|JLo^zRV&YYP!bLPx!D9?#IaiSn&EcNJrz#Gu3FIb2bp{WJ+)Guarho@tTl^^{N zj=o7RBb|j+AL#J34`hc~ch)Q^BE{vdFR6G!kdmsnS_`fw8%x=ZCL3p@^UWv$lU(j_ z{9W@WVZGRV{eyL(G3Mg$TET{j-Oz`$Z5dgc^t5GZZMRFqxTCoE+6Nhh>v& zD(;?}*Hh>3o@<9l6E31c7E7D9MCg8BW&Ugnrj_2P9$QM}-nbl60&LVoFN%p>j_C&+$yU4d0wPK|7l3X`qr8(Basn~j$W1e z+MTRAYZM*wIm8kp7$y{&@~FEp{x@gXWYo5`2pfSE%BFM%EMtheHfjM<5*tEp?#tIS z41KDwKg|VGBl_u*p-)Bnsl~`iSu3cPJ7QB<@qYFC(wHx7P)_H#4kR{}=CsD&9zL8F zo&0sBz6XMCMi#n|=iBm4ayG<1$G#`f5Gz2Tb-(m1IIpECIj7XL3egTuZ*9G%vC5Tl zZ*~PMUBPl!aE2>bie$BjyaQXbz_nzz=Fr7!TawpJ@4Pba5TP8kba`T19!#Wh2Fe)m z{nDFW1|_!Zznlq@jWZw>aK$z&wN*M&8)pK+B0J|iUE&P5Tk7sUkqZu*8G*6z?0gQpAK<0Bu{>51Hdo1^nbp`3y?i-`dpC=R+eGhO z$;O5OIoIuOoRn-_G;lMI)~b0?uC-HSljK3n2$qQdYc2u4B0F&~hWLi_%C_eQ8@9@I z5oV*Y$ikJuJcwLj_tMmyT%5cVnHs4~d=Yp9TnTqm4g)*8Ty)Ix^oiAtrLobRgRtAd zgKXG*2MC3pi4qYtWR^Gz<*?}n;wgp&g^2?$Bp~EPpwUvtRv8-U|t%+oGb{eEyk`bA1@s(OYLxctyyYum-Cij`vi2%qDYV>GBbu6`l_?+>O>ckMdZv0!X$RVvVhrE1HO&bu zC~eMt(5`vmf6GqhXoUp>y`oIm9?XOj@sp>e^dd8YtWVv&*RRm@4u%>F5 z&Dq+OL$AOo8O81MV-kZGb&St`9`|?oJcN?U&DcPRS@s@~G0Yu4mN-4n zQuC$emk37mNm6}VgxbGflkf3$-O5wuSE2=txad?uo9$#agfb;GQO0GWSZV8f#g54(>f}@4>y|rj&3zY>PexNA1};>WR{~#yq+0 zFD`ZyEC5kV%@E>o-3+AW@njp1ghW>at-JQd&$C*J`EWM8(E3mW&maeq2Nu|vGck)f zqmf-+vvrNT^wZ%Q4$u9;9y3Gq)7DH$mPQIZkzIJNCrDc&=%8F_KW=R z25}}Rm7F~yS>A8;iekVn(gJ@O;onjjJ^t!GxLweITc5ZCib&)>97Y04*gObiC=$|* z>{_WJY$ku5WUfR^8k3V_M~A?2um}YWinL@tV9JdMZoSbCG*uoaCaD|2elKj2B&IbR zCW)I2dY7t}%^i1H1lIr9<+Mw7Z+0R1+u!~&@>{O`)iYf5QZ`l8KceUzLq*w^4}G)u zhGDawuQ-bT4KsnG_`9x`;R~CG@6;J^iHEy;zeDdU@Bbk=K&@{1Fo6dzy;5CJ2IJo+cHM27wsZy={tvmJZ?cW-J)7X zMyW*NHB5|-jTBKFH)xCv(}R~Zc`b88Md(>jn;oL{p|MNTkyZ+fj^?z+d{TIIVU-kcj&;M1w>Vj7B zr4;}J@5l+tDgjBIh?sV}$Z$Mzdq@LYnEwa^64hDg;uX~fqh-I({i4iOW*3`Mroz@3 zn}aNPwI}eHN!|2fRAlv!YjlK9FJL%b_&0QoNsxmV z^CGCmbt#1>OA7y~UFzD_v8tIO(67fRkUicz4NC_$QFMk-r-DKo%?EivwDE zeApbz6iey9YnMg6?$FiQU=?$-VmRf?bc+<>cM8eS75%VzF+tm$prw{)V4&v1mzlj* z*r8Zlybc_q4kdIV7yZJG6ok4&WwSB=}3q{zt0T3~p2gHz7ZuZ@0ljVUf zkrt%tuTU9{(k4_Oue0im*hIY=%BL&71YC8f_PxGGYH}0iGX&o>V`CK!4e^*<9Ekjs zE)drW`&aT{*r*<%>Ik0BsgxTR_@@C!n!C8^skAH%fg@Xqeua+^UhLIMpNV>S81-bA z8h2`J8`JX}G6T@@c86|(8>PLG$7m0@o@SdvgXyVkz9)zSf)22bTW(70yk(5CxIE4- zRGX-O`X&G`w@Wb>(>l(Llppy6b%*1>L(7)_Eh|uT zBi`^3(X3*$?1lOdT_+r*(!nX2%7<@(-9>Iua0tYp%fLIdfo>T=EZ6W2JBjnm&rr-E z=H(#fovJ=tJf=Z`brzpMzpjh+rbZSy^`eUaY&nFrSNw=-kIPou{}*9$P$&5KNiV&bTPjIFJa!%|f#{ScC%4j|83_0uPY>u7j5m6FI~;iixsbj*ZU)k09I;gKB5)u)`Gb7vq2 z*a74M@A~TBZ3&0W`@hD2kA4Q2vOO_(pf9#7oBvqyL&0N{SeF{B&YaIMDeWCqA;+Gw znU@PStG)wyw+}rx@An@CsvJq#^{FltdjCsSMYe@IK}D|fsA$<@zV)=Mg`)KNSc`oZ zY?8~3v4YO_b24LM;au%ywqV;)1uxfv(5U$1*$gWc$-|~=6U^WVR+z+GJ-7#;ZIFM0 zzD7W{wDb5xv*WjAi=RMooyuo~QjH)6pg!CR;v``PC?ga08IoD*`8E=YqP%7% zLE4eqo`_BAS>|uik3hhw*fwOc(}ZTXizob20jxqdp$w`GMsoayT#$QhPApD-G; zvalsSq=-0n_{1V(hKsC-pVDIbWf#WDtwK#aZ+{b{`eIw$)Lk)727O!3WEm;MrTRc; zj^p7I8~YZP>tY)!V;=v*z+33axpv3j)`7kJwek~Yrf*9f%pR@d8decM-TWlX@xCo5 z>BA|$E%(tcSlm62Qklp|dlNLxePo>M}YtGUWGqi z5It+)bBrFa5RybOXsck#z}K|CDwGnVi z_G6#Cu-^Hac4}#IvN_Lbv^O!kE%p$Yky-7|lyfJH8ufy0&z$@%gwygpuq`h~03YCw zSpRvp;u|<;;1zlOBtUPn+Y{876)T8#46G1@_@q*c>{46BvS?s6e~2U{{lhcm6xlXF zMwv@!)mCC+<_v`RBU{yi=!$_u%u7U4BizZNoRlWxQBAIww_%F2_Nb2^M~)ui6x~5^ z>Ba1UJ80z-OWkaj+DZhyoB=jP957dz>+HnntmVncfnC0?+y56VpXt;j=P~KaKC$6( zM}s?a;7zojpIGW8_7Ze*tL$Qule4D>6?+MCQiL#L-eRet8PONv40p|!k@XX>q_ZKf zq_(>uKXGVM`*Sw^Pwc;Nn5&Ci4%t+5xoF6$uR(2z&CyRo#(x}hD-p$c(W|YQ^_AvR z#ZqyJbwKo#DeDa|Ch@_j^(cJ8=1VLjtA3`1F)KTUivU$T>_jesPD{}bsZWGmm%PmT zqq43nWBf8mdLI;8gR+prm{hQ&+QSi93ia`Ud-#|1w57Q%RYb$TQu9bI ze`gsh7PNYcC0Zi3AIV@9l3!nN-dS7}rz*MA8LScV-B{TlERi;_D9n^MnQ7w%!kP41 zY(-{r8>HlaK`A-QQIyWMkAWy~V6zC(0TWyR}$BB*d7f z^CB0^(66pEo2JUpc+&-z`~Jf9!JMN86sr4@?h`)|3!AcV4e@Wz(huv2lv*7ri!A^k zWmvD!Rb^Pu>BEZHhQ#cZu}`BH8zYqdvj45kKYiI5i}d%9)uaq@I=-qLOy);ggMZ2J{t5v8zt=l&|KC#Db+15o*eLO{@D0dRW|ZyV?%e>!YonT6+bJb??l;W6k zUMPwZWg)c-Uqc!wHy0{?76TH|DB&QpwM#aZ3O$fEEB?1Oe|8(7iM$iwe#sW`u-qJ0 zyzI>4rE(n+{jSoKi(}PN{R_PO4_#`^RzFZnfltJr6SLV9A6~hsJ~o>O+`cWNrClW8 zFO4@uJp*hha-szTeewf>T|iDQV7^0i;o0jx(X$hbC+ zU24v@D~tag*RQldd{kxOer4f5gH(c*nAOVpGrFd;qhpE4IXaym9hV-(0dZ*HyP&xi zfgs5uh#Xe*T5Fd?XsO1%S}U2_R>(vw)OtP<%*Q!_W1gX|$;mADgc%jGa%C^Sn|jq& z(VHA;5@D^M3E{8d40~O}<_-_dZCZm-Q=}4LxRf=h-N16CIaNHG3m{?xHh~B%=Lxe~ zg{%q@Od{~N1ngaQnvv@_`n z8Gr?!SVp&DkJ)8po)F*qxcE@kA9&t3kKM7e%`?B-3$nyLJ&p6a7t5(;ftBNXgeWD^ zvz8;NdAhOgFx%fAu@0Y_EFBehIwi7Dqh~Rm5~VRB!g1slowwgZ?_slcXKfusGu=dc zXRiHAP7c2!fP8n2tqygL2JC!gPQI7l7|OTd`kO=b?YZt_`f-_)Z})J!_kCQxfRpcK=D~rl`{VK*dHL0$d{rOGr{|m7?R@s2Kj0L4>`$^jo$7NxQcO=7Z*g*7G~E4{ z@pm=igvYM&awp&1-w%QD%a7z!FlLSf<7}tUIObpoj2n(FhF(8aH|WGqJ#wSn*@p(i zF*h-J@x1b4r?^ z)lZdg3TAW7__$oZm0aUSS}nICeUY>>PV45ZIU^xywQf>-*J++?wbK2eiqE3rSM#cDq)dAm*2RN5 zK3*!f?8+G(`}hVer@h$j0ossR`bXhD%-Ya?EfPY=h}@-$|5MQdQ(aySb=yn6#eA9y zBPAq=QvNv*TQVGYyk5AqcC^Jzku0T7mYLd70sSxt%ZE#gK{1wq9iQ zaaiP6+ab#r*wKDW;F)%=>*0Z&Xax7me5A)l-Koma%L4~|iw;EpQ?VNdY0D4B3j;gW zpJ4U*eobb2Rwcg0Yak!_^xj)ad z98ONmM@B%kT*|qZt0r9*HoIkAa>BQV@p-VS6Hj+Xa<|@&pJ!iPFPikf@ma`#ca`avK^s5iZPnwp@ajMeJ&(>iI&B;xKZ^SbhBtEvt)P`?$eA#FDy@;o@{~ zPvkxtM~FIeL`^61zK4_J5NbYxuG!Lx-aQCub#23;zf~D**LRcDHyAxDY|iS*mWebI zOP5;iWzm6k+8DV(jQ125yvRP{o;G{3T}j?m(lQYI35ZlqiV<#aRBSX!&jCzpYjZfV z6`)(pn-Qs%^W_@uTx2l{%MtC~yD2!I0k+EoPowWhX@^#5}BpjKp4gQTGW!MJFL^8aPBrJNB2Dw?*?tMa(rT=y*dUk9`(ys=ZP&m@5<=1GF#mvKJ zC@Qv5+%dVtF4(FCizO<~G^yxehD!CpW2AGhk&77NRM-zWz#Jf*j7XOV4SSW-|F2xb z19!5hS;B@gX2&>HAJGvhXGFX@Qdg6cp6l~hFSVQhh2|;Laf%;D;aMymfx}CEnzKSn zaqH09Zj0w5@%^tzqcS@l25-dd5O>2=(t<<_p5e5#+J;8Pb)IzSed@)GA|fG%amv9w z%aYp`D-1^#G0lXx)FiJO1q(zsNRO%TZhIX^b}+V*zVkCS#6G(#)J-j;%~z>kObASh z2L|Yeiu!&jQBFx?-bj8sCh7wqLh+Ai>8G|F=El`D2F)=-DVpDAwM|pY`QdOiK?>~0)oitB7 z^!d*+7Wm~KG+$%<)PfLN%fQfeq3U^85FgN_3zmMWRDbMavV5mUT1Ai;#5pT>=DGSLRr=-Iwl=T1ajijAl%-} zq($VE=*G4e`NN`^Nd~$yZ5omzeLZH6%nxR48`AtEZ~gjON@avsGV(Y zNt~8r{ssO?hKXeONFxG>e7cO`WL4Zq4$KSGU|h+zOpo;7HJeSq+CT<-2%mD2BC7#c z@;qnLWi4kfZ-s&TaNr#%bmx;bEXcX68^Jy%(&NZ# zJ3|v0DsX`fxm`i3@HdJDq-C2pD@0g_P2lYXW}7`0OIiR45QKkcX4R9!XmAr(5Z<05y_A1F37AI{7W zT&zP0?RKck*JPyRs(Fz>klr{1frzN)vLx?ukLZ$#pT_;?9Y9PE$|A1{uI=Lw`00f= z^EL8@{rVpolcu&Z^w3+VKZ7I7inMV$d(OGb=KT_$bNHxA``Pv05%|y_@QIYXf-N+WiuV33EP z1Z8oC{+;&Wj>}8=D2fz-#M{i41atM^f2qF0KP=vLJc7C30s%X6FXbQEtiX;jU0( zU_#*K$X%p|ef2M!?~ofYvvX!ePOJ#?Tk~!5YJFH4>Nnh&ll%p(HS$9#XKv7!tXwK) zBRhIOa3%h7Vr+C`AJ-N<^Pk_RG)(W8r`>a&8>r!)#oB|34?Jse>bP=y%Kf;99wvGR z#}0*kO+1nLog77ruO&~f@dO@S`@Zji&Tfc<6B{;F=NIxRokv)f!O~U|`zwd%SS)v# zXLXpPa$>l@ZWp!Ym~BFsrZt;u8A#m{Jtc++DZZr|592T))i{`3{3`zV%C<|~nQzR{ z7Lxf#c|eK!R|WdS$vF{}eytO{@~K>+6fg1gH0SyrL9sr~(a)RNzAb;HUG(bTm7j*# zRS6@pvyCfO*jM+$v{c;--OK*iSNCF0Xn;Xy9qilM&G;tjUUet>+{ySpjCFi<`RRRp zN#Xc(C_Qst_s?U z6+bT7F3i*}0Dt;Q+8Aztoq4``4zvqNdAs0{1DH`-QXIf=wQgk5zPguldh)yL_N40e z44lHcjohv{GZz3^4)I-t5{gbs^bz|wzRT=mStRS8l7;5$+RPfm`Fea8qkjz9E%7Vm zRVQ7kU@o0J{ZbNzmK0vocc9|wTwE5a7bC0Y0Ef=z*6aCflQ+5T!i}gop1@p>M3Pv! zof}bQeX-LuuDOfQ=2mXmD7EBI7coS+1l=B`LAPBPdRl#H*U(13sYMiAf^!pIrhd$J z>kgxTJRwl$iBog>svb}02OF@${a|@~Mil*{o%zo1EBY{!+hinr&4~mScR${_7Gu4- z!;ENtPYynk)H;%jl-uy8PoORYOWy-b?8fgfS`fuF-L2SGl}lh&4FK*Kpo&`?5Y?P> zgPhunzn=Z4i?umueH6A1O;__T1y^Od9EJlGbA*tE(65l$hK5-?`HhQgWm~cBS~#0) zvqSGfCibRtxuHh$e_Hp@_CnMPEHYBfFF(`zioxZ8g_263H&{n3~eR$vaAT?!4fnkYE~`+xm2#X+muR?#ISjTwPP=wU$L;G;ydsEma^a~A|AfuQIqwaS?iAc5OtM( zUY%L%iFozfOZv?lDb%+oc*CbFA<>HT{Xu=lu~tLgwW6?ylxXtZnp{M->H7X9%$YEA4Em$aM4G3Osb{aXiq2asWVlX|U)L}c8;ARx@!v(&|e@}iTJtlMMa^6s^# z3G!{O;Jw+NC*16s7G4pe@%hv2Ox8Tf&|psyPJm1$?>gRR%G_aXTeshogBM83adt6W z#!#{JPa??1ECwDo!mGWGG_ui?6u9>amu5ZOK?5+KP}`;Pcl!xB@4#U zjUFP=mM@XMHKf>>*w84ogqxZOk*Znk^N};y37NI8(?L=9S%cO)rWL8lTtZSycquV2 zps?&xouh+q*|n>-R~fy~7l&%y!PY5qUl%fBIaDD_X)dL<7Q*A)$p***Oy#3b?C#n7onAyzD}uP{j20ecAIz_U84MNI3;Tu(e( zZ)Bg6seI6wp-KN0k{#J4fUdq-ln5U4ZJHf%oPiP=s--sBH=8?>qP=N0OK#SBR;5pg=Cs!P8a@b< z`{5q#ELwhCO4nydxsvQ`%IwWK5~3}ZtuD`5buPLKIUGK>Y`gEbTY^7jeO$B4%f0c2 zB5sK%=Z$eMjPG+!pcDIRKC;jR>=cgZs`8U{+uHLIRikljTbvXMcg$XCGCBiu$rEhK z7`r%C_f&G@V^BudBpKtUi0~aBPon5;qmXhzUdI_BGwAzmO=a1Rzy}=@BHWbap2WIx z^C63fTzGZCs7-2ueCxzkM`yK|JV>4hU}zMPXQ=`XY$5u*5emE(o7+q)DTEplIa}dU zb9vdJ+zq8-ayF+lvG~1ALt^nDVX^m1flPc`f885ihNH**?C!+p{f0w8CFq&m+mH`Jni>a2Wz zI>yf`jfGKs_H3AG-l87* zF82sxrK~!4WJ|Zq)M}IxbdORGvUL-Z_!8v+jr&kk6Fr^#NfhOvh`&yyjqY$n>|P=Q zY71VCj8C)tgTKuwZJ7Gb=6pmY#PW)#QCpcCKR+}$h=g)4_4@LPz1I3dPC_Pkc8tm4 zqSXVz(bjom_i%jiPyUaK%#*1%j{_qMU~dd3VT2oj@PwkL5dV0S0!l#h1dK+aL=pd8 z)5H^YcGDv^DRcaIK58;wRJDF~+2LzK!kk;H%dJ(V7}+n710o#PjE>_R(jTbz#Pe|^ z+v=Z-tH`0Z!nxLHjPLhgPJe8ZgzQ~ciWAN+ci7zIaF;+dm7R7Vjd!S=%SF2mEFfG z9nx*sR9!>17c?r7&mV&D^84hAQqBpLO#-al zFQ>}bZg;^|8A|)Zq3hC zyy0@I>~emq7RprV`@_O=DY$IevPtWwT+{2tm2H)mt`&vNJFgFQ?Q|*Rd!Vs_YyZAK z;0Y&K_212v+RYV*&DP=%@3Fu5!Ahj`F?_B6=wQ#L5Cdtw(6WF_7@M67hCv$ zLzR=aP5~*{{SM|vU9H$ZBKjybgcN?0C1CFdaZsl6|20F){b;2C6{>+U@rA55JYhan zu`Lt|MaMLog*O81lNmb_IAKTddf0+Bt_QzU#V?>H zFmpDt{BFb(s#31f#3uhn-xe`pOKf~0Cw3N_d$?dRnG#pLK-%EBa2`Cm?vI{J$Yrd1 zo6WN~=|qW3WlFDzoxXXK4lG{nC>%BxHyP3=ujGEy?}r=mATH-^othh)GwQ}si8mRof%BnHRD&P)uB$Ke>4 z?ros2Ldz%j>L7t7f zdiPDSypIsv3iE}ry-D{YlHq`4c$o|uU;Pnj@bEMV@^Oy)5u0}Z;p(Bul)e074^FBQ z3*4FRxnp|foZCIu(>HfC?!pRjQPpIJ6eQ2@WK&?xsk|~%ga7(akqw&PZzALuGBP0)xywmf zrb+l)6hdR3-KA~XV%gCsE+i2|CS%zASlCwF#_RRv=5IF);dP&kh51cnXegB5>J`C? zOiR64Erbt|r%U7;F6NyqRG6i4H&tjZZp7GG>dj{~j~L0mVhgQ^rboFIb(S584;{6)lzGqryQzucr` zzu6GDphaP`nhLCyumJqc$0kLNWO7wIT!Q_`dV9XGI&Y%tR&~evo|2ibvX0R)hs`1? zQdRW#p)O%_4}Icz88Hf`#!XljhisunR(v9{$DE3fp`;Fb^duq!uamr7<@bw|f~)AJ z2o$oJdV$$t*9CE`5lK|+%fQUxbGD;qMU)Ko?TVbJFVc>5V%0;MUJY9{rLrYF`B^Ou z8eM}7Nd6M6J%Y=aEeiv3p(~LsK|1&Y0VMX93^U!KWl3|>F?D9(L9f_6UT=(*)4Fha?8&yN z41W(`Vo&xI$w%F%w@TcO zH*(U)+C_>O1z|Fx)wTzb6|^rpU3LY<^nztAF|yX;jC>px@bE6NR)A+pfuAt?u#)=H zvc3il!x|lhNp}?C=CZZ2toJ1rB77_?%!K=0;{ZKb>oLzz2K_PwKUrTYv`a4s&$(;} z7mpGpl>WAu{X3ONRS3JdpG}(#zQVu}6dgd<+TlMd_D}wPIu2ibLo(KzcRrL)ObuI5 zIr6z15nFbn0=!V|0}ukf%3~E!>Dnk&4)Ci>hroeqBnDS)niC5o%(U3KINuYs_Q@PD z9LxU+2{SI{!P~X&)iSe;BBo6LL zT19Lk+VO^%JL*eMjTWRQN5`ZmM!iJj^0G>{Hn2`(Cpn3|MTnHq)Ix;ZM;F27_S$kqH@F-i*_~~p_hYO9d3q1t^#q{V& z`Fs4^GW8|t@y&LkmoM#apl0%5YDT&O2m$;gYIDDAEFmtmFZ9GvF$sSuu!W=HskcZfAdx+(g z7@Ud0WHCq`of$UIK{o(M#O0cRtK$Sce2TCZH(_y?SywFko;O}nsF11=uLuj>n{G~& zcIV8&0byWl`+Fj)BkCjzyYU0-iXDUdKzD51QbbX|Y5a*(y=3wCjOw1Zq(bud%=PeC z+0)p7Xx|-dz{}k*A5mnr@acxzZuYVD9u-PH4c+V9G#ZgnHn;WJ%`LY@I24CG(1ml6 z__?{)V&*d`BX37Lj0j%wKH|Yv=^mY~4wj%ZS;R;{2Fpz!JX7W-Sds~5Jr(}k+HO#X z?&m_U(Ja;fn6jc-DiF<5Jtys=TVhF7JZ&y-8(K|AOC~a4;>i0DT>0t2hJU-C5wNu%PMY~8 z{kD}ReKiY4Wm5y1QA?TV0hyn2y*Ldt)m`0G7W1J#x4-XwPkNGl!#7FBu1{oiM(@;N z;DOYyp*(bTA%_X>)ZnhKx5F3lf)6Mvwo;p}RQBCsiPnTa!#Pbb^pa?55 zhzBTaGx$qZ3@t%MPT0dm9(-I`M$sBV8xFt&LHG0-5l?k{DCplG}PoUC-*AA*6j$Yt8ZJvy^A)qnv? zw6xj0%QjlCocSgE!?1z`(kKcNtDtj3L(aNSrPf38gSiz?bk5F){)t9)x!eeCHeFId zL24brC?%2yAvlGRHDGi8%<#O=940b?wMxp9@yDF$MlWL3vZ2@!pQdEJI#|paR+`>y z-cyR<2wCr?LD(_&xc`!g&)`hGH99M74)Q@K!JK{#@EowoblB_!X++FvHWNr^93CSC z75XAPAfrgQS|$=ZT+~}5hWJWQ>D{$Qse-8?uguph zb~)2iM7BWn#);RwgoeK;Q925TGE2Qo)7w%BT=tqB@*!E6tGw2kBc=#;%N)J-eVrrU zZ^^r(22xd9zRLH9-l6jhQQ@XfIMR5o5WJ5+$6yA6=Mtx6)63cIKL$3Un}_R#&F5u= zI4LmfoWrIFM#j#l?meEpJhN5pmA~6q*CN^YMY0j?Jf>D8A@IXc7&1ss=f?l!#HWPK zI7Dr6WhKJnWvm;Pe41;rqXwh~P2ET;=J)pcVpTM^hlzqFm4cQ5=ZhwAiUUjgOTmwr~$1KH5COghK)w z%n=@tKhTV&16hfG?P{sgE5-2wN`7RQA=N`--=7jYjz{evr4r9%(r(<`_O&o(6QK#p zU%e_VZwU`!kBs<(YE2UK2YNTHiCvK}H^!RIyoIF|y8wyzM6STbKa0k`UyP4(5)Et+ z8i?k8i6=+z%o3+fyNmfGGtpxH`KCZvIPegewG?}*ob(R;wuJ@II`F;h7x+2>W9J?!b zW5l%eAS{3ZChS?XU^#c-N5lq$cl)glcis`%L=PV0`@#Ub7zX3}+&CxMrfP-Q& z>PgMdISxpFe6S|xDgOVN|F`r1E&l(J|31EbkKZ%+&B-0jzfmst=sbS&3;5&7S{AAM zm+E>syy7MgSP62i=UCy;Lp%=oulsl*0*|kP#+&@7w8l>1=gio#FqiT4jMGH?=S_s@ zTrE13*7PYA?{kfK)T*$#n~I~HM;o{zMhYT1d?PAzr8_1sV`Uvc55eWl#1UVt(f5Em z-T>VX)cM!V37g9)Z5KKoyCC^EpO3ns4x46b6fX1zEN7pD(KNy_pNwoX-TrrZkuV6EVAav1^|}*tzJ_bx`<|P*27_|&2LgHwdg2u5A({2_EWOH zp}#()BK&%P*ld^N5?sw+(jSaSs@Vrg4x7~3@^{jMbS4`av5&eEkIfX9HdgJtHh^4} ztgW$`IwOK%{xuR3GP7>FboYll=umJ*G{-*3E0MI@NgEmyDZ&^K4ET0BCZB)9AVP^Z z=u70NysE3-z(;^;l$sAjK>~9nqelO^8pcR4Q{U#ts&7EYOgI{=evyQds1GHBp`QAx z?qHR*1Sc^wt$Gfr=Mhdb+2~w16xotcCf4fJ3Q#2g$$q(q+Jy=dmyknAjb+H$7N?z@ ze?`9XVGSqg!Af)OEs_gwj`3G;sR$xunFCb2MW;x>Zq6(N>8Xuq;#-7@Cr-`5`^|>F zpxYDft1mo>__=wXF}Np!FM5bao2uWyZjB&6U+5++X~*JPzdG4glWg-O+uX?$A-vJi zt<^A>Z1a*>a zi#CfZubfEEp_-g`e}YRDOZrmSW@~Jl*x0sO?+!(VVrg;&FF`K3S(j`oNjA+$Hmy!J z&Fi0Y9<}5qn`S1PYLZQnWK;Q-$vKtXP5BbgwP|{?sVLbLOE!fR{aeD3ZDg0r3alBk zsgLQ5dm_6)FHO-99ff&6Bu{^^TsR{76tkRcj6UdtOQQbZj5A43b+4vSvauN6CULi! zoDH5tkf6GpEH`@EBm{V&6UWh$S0i1X$SPo!N7-WPGSq!_sxOj_3mk7gud-R%1xoeI*8pzBF2Z%c1aVs|aoq}%3y{(M zZB~s$$r{DuDjt)Ba&wN1)ykXQW#@)T1dG3q@f1O$#fiUq!TPB(4x_ZauzC40+LOkT z%MPcJb;k-g67={m(KC{998z_RwRYmr_rrI8qN+m9eybZj9kZ>ChdKIlcN3zSVL2tW z7Qr1=f_2O(&sgp+CSA$mFIunQdOvM(s)-D>c#!?qV4%^{zJq$0QlTIi9-J38%LE4T z&P(7i4^E9&v^kQt5LE4rXrc%Iu@1w8YslN03QlmqpUSHoC1LyTO|JE(Y6$>em_%K% z#2>PUudmka?p~PJSDT+>)?Xp3&<_{wn}^DXOcIvzlmk|-<-RZJ0;7uW%YbilJk@jD z>n6@((OEJ4Fz$U*?v1Z3|Sj=11|5blhd) z`>~2w{S3)qZs%L-uCpPv?6B>XH1On_W3$dR?*>debE}=L$6eK(Yd_lQbCOydI{L8f z-L#>}acLxo35+1U4f<+A10VC z7Ovt{6$hy`w1+e_J7nT}7;YO%pw;g+LECL;K&x1@-3GSFye6eY$LByf>wkyxGcuCH z1DiPie}OVI4CN*GH~1f+to{g;Uz`8`JC-V@95w&ICHC2S%vzpC0KEG=L8jvL6x(I2 zJ&sEV4ch!E#WG?({Vh98TQ3+ICfn$B(l7+sg9f<zJ=FVW`(0YSyI|Dn>b3SH6@uy1d8JgnwYw;zC>gYIvE%32L2f3GgNybc-&a3knveF@E2})3FY9}>%rCzuosA~SUJDqx z0gCeyxo^rGSkFVJ2ykNaf`@B?*MupE%3IfOlX-DgwsBpppBir3zPLG4O>dM5iTLa> zWy1*yj2bmAW~g9wkiJ@Z;!CPODQ*_$-%_K+R^w#ONXOc8}r?AIo!^!vaF9Lr3T=da}E`2 z4Nc}}RF}D5Kx;OaoJMZvrXx3%SzqSxyb(BC$=Sz<8PpRK=I{{fX>X{O4e;?YW(OEG zk{GqFJ^ZG+#AXYwe2xxq;efzq9;o*$II=cKv}s}g|8CFEaon}fn?Eq&=3FBcpn zIXECYS2T{b;hFQ6gK6DA-(m>9g;_r;r;iP&cogLSde#9+Os=a)McZG)F_N<4UZ zjQD8Z#9f8&3!kGR0KN;Ds1I{}Mn~WXnMWIRLI$>oEQaEp9A@FKIkF<(P^NEbro!~4 zl4yPj$0q9%$y!3z^!W+%`PlK|knRArjBrdhn!9tpuD40lf*b6DlcnJCA8n&r+bGJy zRZu4|*g$4SX#+$;kmr6GXx~luf;y>`j8v2PBTQFz?aZfrT5xWyJ(H=fr^q1p13wQM zu;_RL7#y9M!k~0rjt5Tw*Ga#8%_QZF4A>=uwWoUH;dSdF zNZ!twCr*p|>yZdf44Lmlgu85u7pHFaGKAInYY@&K6CQ#yI`@gJiv0Aad6-!qpCtDK z^wuRi)CODNN}qs{lkbsj2%`yejxrWGok0zz-wW#l8xOC!byZuIz1Z50s0)*g7V{3k zVIYou#GtDk%R_g^)#8a_A$~XFCx|duJPlb*EN0A4JnZYmiNnrf>2HZ#zZeS#(?hR1 zxv>&}jFxQ_-}69sVyWaROv?rgEnu_xjg+UxXHS$5Md?4t2ZZNl)6YyZ@##-Vl`oVy zRh}SkOM#Cjua%#Oo8iVxh=~?$6q}>(oJroS0yN7ee>zuI5{kT^l0CH~<9ep4=U5X2 z8(Leti)VA6H&={hm0O4zf_*~}NMIY_1GA%Zm3l-cW427j`NEB?{Jxx%nTiz+Y*Bh# zcx$eiIVR?vIWBhg%#&m8nHR?=R%>oydYb#%eL{%m>C1E^`*Z(`hUQp}*iI(A5n%&T1EI_MRW3D@l0&aAQ5iCqwCpZm*F>(-oqNT zSz7X8A1qB(7NkoFS3uH78QMZ%a2Q&=7=cHMt$RTt$T&pi>De}svHHup zr1`*+I36jRP5w#Qc8v=x=k(uUk=>}fL~4E5-fVNLfe40mp7b)=bq5ZvQ-QIux$H>n z0yH1V#f5AM!+3DSNc*c#(H8DXyqGec%yzrBWlmGGIsR8lauAFrS&IwHb|zOAx<0UW z#{2QJp-Sf3O#1!o&OWjYJG4lQYfYHUB5OoeBTC64>OGE^h`bzWL#1^+Fw5Zt&T*#7 z7(4EJKF7l2YZ;y6MjKD}OW(L)BNTw^Jkn{2^TG4^_LqfxK~&KlpR=4qlL~T*CZuAw z?~#JCM}1o^hNvY>sc%bw0dGJP4cFKrS4oJ3c`iS4mj_FV6by_051dW*o+Y_Gy)!Sx z(Lc;(=9Mw*7n!DxKg@=LIVee{T~cUF$LV zLMmFhIQ1~Xfwc>09KS98JW)B&dGprcpWQS~j_I;u9hfr^VY@)40Rh-BUlR3Wuow6qp~`#>)0R zQje_@3a^?pbbEYYu|J;Q5FZ^(^LQ8x3kKG@*Su-HBOTgGsYJhf;FqAE+`jZ}5%Vm% z@jeWVPOu+e>t1`j?}725aFe=IR3(ccY+#&*5E+BtQ5H+G?hI**Swp}DUxCj##bmxO{opJ1ILrf%ym|s zPSmm^srpeehY@5dq2!;7X43y+cXTB;sQsbz@d70Dfy>k~itnqGC_1fI8QCD^j)s?pXTr5jXtL zw$#bN-~&6r!NlO&O&7)*5=3*V<^Zv*0k>fW=i_p8xkwnv$)5qlHS+Y2OL&UN(^h#} zB~J-?x=@}L$?WqbfMyusq(1EG0}by2;}u8Zo<=@gKcJqFbZ#J~T~A!^p5t1K`iUr5^KB;< zwxnaNC5%)&zdG)P@kQEMoh?ET>U5ju)or=N)PZlYtk~!aw$;m?Tu#@9&LUKC8tCx0 z&%jO6lwmPUFsc&z3S{XEhj8ppD(6(Kw%NM)NMYiTpxjcEm3UuaQ8dS+5?7e#DR!{yo+Ah$|Gi5Xr%u9bm4-=qz4axtG}7RPCOz6XeEdLD*ik~fzr=Jm9p*UrFM@e}RDi9?B+$944Udn@f4 zuzPo^|2b@yBS|LeU1)yU0URdw6-;Xi)rZaZkwXHJWqIrFSo z>C6=p^y1O7I*)pPAe2s>zagX{u@>4y!3ogEDYJb7w;BTOY{OZSV!nYwyQCeEoDn`w zL1IdgMwqDNyd#AWs;1carT44_fb_6c^hVD ziyZ7cH4)X(hcH59S%S@e>s8V+_e*woE6Sr4wn!H;7t;*_5H_;Sm9N)hB$3VwyzQ%h zTP_*n-44U8?I%c|5R=Z1?4TFT=KRN6~?~l32et4Ve2?eRcA_iNPOa_ z;7zs;*Iv$$7gMUz3{*iUxv2?hjCc~R_-UANV9GIJ@!+582KTP?;1{IMBaT6EJkqO< zM^tcg;e~ssPLC5C)}3m|YzBuiuJw}}LgtyRGQW$xL{ZPwAmI%kEsGRH@L*2csV4n- zdbWX5upJK13AgA8)D~}YHKdnu_G})8bFz*OaP9+#eByTwKAf58C>k3dr=hC}Y3~md z#HL#7iXmt(Mh2ZF`1eaBGu(D;*;;?^kv!L02{5+8_QENbBK&n;x|P>?Z7@f>Mt}=B zbs|DmILe(TFUkOX47{*N1R4(}JavOXmxe4vcv({L{F{x5XL1j|-}~1*SD3?}N3-{p zg&|@uz6UU3#S(XTTg~(P9IsD(Lo_RvmEEm*oKx*4 zeCvwY9yiKOPV8#&%`>4PJ}Yy+)F!ebByu?&Dq!kxa4S*b1SV%q^%Z@P{i^-71F3oT2vv*v7;5?$Bu zmV^wtzQ`5#@jl0&McEtu)_eetzoESmw58!e=}R4_taE!B3ZDNfc)H+}CXdWz$S*OU z<9XE!{IT)YQi4s#|EVP@XCLSh0l5V80S(Al_9HO&Kr~P1OfFk8#+;eS>y`b%{7g=A z74OCULG80S150sMq2}JqGYZuqZmMU3vNn*pn40J9b&O9qBED}t&RXcT`Z(do>6EJ% z_XqLnAlF#=)c#J-!h(w3WowJ)KL6LiI~%833;mKa564P&&N<{P3B1$(t|J*h?)vLJ z%)Gcpw%z``>1HqsTx+>Jn5^ee*PX1#I+@dPB5c;ApXhxv&zg6RKUu)vLX4F8DZ)g_ zYW7=R44M7bqI2D1&%Eed50=j9or)+>7iX$t`8^8?SOkbq{-x5lzpA8b-^XUT$cdsC zYBv#<&{v-iM$Qt%i_c1QKqU$yB`LH5$MWKeAJ5e0Gqc+Sqi^&@D)}Te{^w#rT{|A< zC>8_^*5P1@{j50=W6WK`ah$48ljY%0)g2Io3ia0w%5acLaO!|mD2BvT;!rs%5jdGQ zuX%JtHO17xnJgXiQU|n3f!S=A%@PD>VrxAOv8IOD*$uJ6hWP0qc@3lT{61@J!zptx znxkVb2eYO9<+fOtuCQs3EBG-)ht*U_Jxr0-)NqQ6X?M*XvZ2R{UGu zC~<=}9}@Z$zl4+R>c+x#g^6I{KF;#lI3+d|<;2DXde`rlG|UfF_d>Waf$COzW1_*!r;b-Fp(373U-_^QecbjsEm^K()>L zZv(7hwTY1NWpeE@*@c;ACbjW4gr|!4lU>4Q2;6)eow}o40OkaJZgH)(GkJfESL;Y} zac8n_3kFut1r~4VD6p1o3B0mtG5;$qb!uqsgc_ASZ`E*xe9+Z9*Y!$rzS9@ZUQj2e++NO~C${_MhU3aGL7KTVcLEL_xT2ghlg);X`$IR!+ zgJI;0u8rN6=1WP#HLeem%eG))(U(j~wq#u=?ot9$b2TweT<2KZ)$@1YKu3OZ<(6c; zGjK5GckNB|y0c&U+h4G>tX0>^kd*CcTAI0E#xHZfjITQT))NK4eQV@rGlnD%){RQ& zLe{V|F0fV=C+iNtusLVMN$xlsm&9V2wT57#m=LjF1BZO6E+*FMPj(bT`NRg|J=7Cc zkW)%;qTTPwiJh%V62Z)gkV{ZSmMk2^I{}Lneiuz}CkYg(Po89hr#|th*i{4$wLb_> z)XV~W-}))s1MhuZZa*n|T&_0N5(R~7Q@r3lfl3dsKYacz6sO6>Tua|jW33z%hYfj& z?H<=2oW^2tQR|T#qaJfAYu8%H{dMBhSql#f1Y>C&_0@bPSIb}jG~&c8l@TT)1iY0O zhK6{|t9Pqpz~0ayJR(fU;L%ST*yF8!vGYQ9u+6;`r-g*&FEiH7w(x8ho)j> z?{|ppZDCSdk%)E%pMMi>B9IlVpQ2nx%|N6p6Gp-m6qu+VgC>r9j}K$rbOhr=*Q1F? zOA?o`qAw}pzb9u&e4q_xw3y@=e)H{9`fAeTETg0z*U&X`gnHPk+QE0n z@*}x_J5EZ;mmj!!Li_l|V`Qqu`RZgjB$zzGKJ-4+vCU!A!g+_3m=&35UOiO&*&?S^ z^EgkK``Yb1F8X*__XVhb-;BEpyG+w!cK@q;HF~U=g=9=|>}#N%!uXsF*C_ul2;@ZTgoDdy$Q7 zk*Bpu6{D#_H+m1j2J7rd3T`4P%%KUE5krw>rUHeaSKkzAT%K|$tBdDmW<)$zWqkrV zIqxE8*| z#U8Qr@u~H>+WxF;`_J&8ja%`<=y6_aHoF)(>zM$yoSnj&`K|(a^}c4(#Je4kX0K ztDx0=#lRvQtOEq(^S2U;k$i&Iii1|=Gf>bbf3%P@ zWLVGO_&|+>h)w_#up=wmNw)1v$h0Yn72$2YeRq7qaQ+1O8P2Sggn)OHiN?57hxpTo zbsc941+-r>*LK&Y3*euUWIDOeCmp~Rqhp+|dQ-;-T9I!TvAA`WU!*PLoZTU<5Gc&w zJW}lgjJ2!_Z=MvB<5!~lsC)UFshNI%R~zu--0)U8&ikxZ?-zk{7fZ)6b;WLntBfKp z3wJyVwY%K|8dydH8Zt$eWAuo$)vRjyQdC>x5G@ z=4_sBw|$1Tjkg)BA+P2GIb?{wqHS=S2;{(wPA%a!ktWYt$!EyRix?f(9j+I+@B(Nq z9%}V6ZA9+bpD)5iB%~vGieQ*Jef||_uEossp)zM{86b`0f^CrX$z?aY?`{Q0#!CDd zuxP2bRtzUMB0p@**8aGns|>@1)V@Rq>q$DmU{UfpMF_2(c}rrUsci)M+x_BEL*!O) zNP{0o8KntSk?naGm1u44@*t@#<`VLYlbM|yy*x6RJR4A-9Rknw)Zqv(|KvruA3Ci69ARj-O8k@;aS|O=-x3b26^!a_r;J{ARPcEcW`FkI40`_FY-+z5_fX z%6)M2IClUTWyTAexG%~w{{?%-*6iYr=BDawR`#d}rjOJyFe7%L*^_+Pt6F<1-ft!B ztt#An>kD;f($vzDIp5rQrwpE-(~tAbGdxSkZA{4Ib9Bv4YF#5T)1dKNbfxl~RBYcfa>c`eQu;$T}U zyAH9LW$8`y`mt7C=2#m5v;HzI79)YKL&W?cJdM3HN5(DZA z>AOE!**zz)v;E)Mo_7>yR+nHgL*r0J_6&@T`WK^j^gg<5}aCX3tq>6`CKR&#$3i=G^D%2{02;K6shNEadF4U-E~6M|yYc>Lx=@^o!$ z@y%lu)R&kCdF6J`Oe1Q29%@V{d&idU`UbP!ewc4QXFn`3SJ)2=O+p$%mtQiBa+77z zjcSr>0)TeX9oZ>5&xCmf>9_-8ZBH)o>RjWwmTe(K|7@z{=hr&zx$~t|$qM+-xQNlt z-bG6SJ<3?9MUCeG10vVH)PXA!IYyDTZH~u$Q#P8alJ44e{lv&aloxU?V#xo;+q(cp zRa}q%yV*^$z`_PdAPPv7plGB<5laZvB(MQgFc3*Vd04BMrdAPlK`bGO8(Nm@Dy_BE zueNHnt+xHPv{j3Ung=EUQG@Wsgog%A?W`LiJcIy|{eRBf-8^ilef;`IH}~F|Gp{o< zXU?2+<_y7PV#@tz3%Mpuv7Nbirzu$Sm2_CNWAHRcR0sAjP>T=G*K096W}(WGtY>!2 zm@On_{5w+1)J4%_0Mc2Sq&bT=NSW8D*k_>UHeT{?7SPDGnF5C~k7a|9Is-)n6re8G zKHFloivw`G!(!4~U+d-F#5Z#jg~=&c-XeNE`TdZPzUj0Cr60IEa62EA8*pr+a2i_gs!&9E?By2lyo}ZvFBg z_MFBB4B#75MJ>aPcB6Bp4GH27eo<9KUZ|d)&TVuvxh!!1ciSLW2a-dn5*^~|!1?u| zfk#~nst6_nOQ;|I-DN%C!@)h>&}<8)e%@s1POmrtlK~_qKqiyQ6+zL!i*u!Cozab0 zbiK721r83qWZV5Ya!vywhd;+b7-u(fIL$$Dq7F{<=OjmhQ*>~u4o>#xjEMx}ZWidP z3HIkqK){j4A*A*!rZn4xjC|=OP8@OnI0sdHm6fnVM`xQg(kuxD%Wx}XnMYGn;7D7O z(xY0G-&?gP|L?0Mb&8aqPWkL)UYQdqKV8bV`YIbJe;MT)vr<3`6>F1L#Q5zQMvmP- zHN$S?ICyXlS=TO#VTYQ1Qde5DF~fAQM)|~U!D!1C)|kAI3?=E2VT@AY&=OIjK?8J4 zbriN2;S*Ftq)lB)HMiMq+;&YIlYYZfQmC9AIT+!w(4})AP}b#H;)((1u1I~^XJ=$P zyzz(zz4}HaU6_`>6yoIcF02laIU4Wo`nY0IoOhwAFAKl8Nf;$a+Km)OHCQhw2ve?v z%p=@`zRh!W#r!z$6+NM6s|!tNKob#ifzHs+H6Jjna{egHSGt)o;FlTw*L}{wQW!U9 zU|s?O*w6^B8{+V{UfUg+^`6wscRjUfkbQLKF&W>m~G5*waUwUtq5i5qMgsYD7iG29A=rQ^4do1NU32 z{${s*kA0=T27wd6-z>amDqgsaSpyw#P1Yk8PnQ~=}!O54s>F}qySgN3Hr|jEYukXKO%s0A*eR6 z4_Q+{9PW)6W2Fwe2mO>v*I-abaC;S}`oF;G;=Hsf|J>HhhGm0`Zh(~Lt8Y&M1#?@4 zm+G)>_Z?2irBGS+HyI^3Ds6*tp6#o=n(8gV_akm|yjo;Bvja)*+*ag-Q(;p-WVDn=Bde7$QsukxFvxi7<9I?SckUb)Bb&UcTTwJz1DF`o^905K~$Qb z)~@cZL;f9)?n}5+(g-rdG$t#pma~VIS^k+$-YA?-v$`H$N*8u9DCR6c!kFtwt8-O| zWO};CGu?CVbWZ^+F}}=*D>5>?X{dJGP=2lLGWF9oq?AbI22#Szgfy#pEr{Fz65VY0 zTBmFC3Cmc=>^6k;G5$Cqi1_J)9_Bhi4^(|RKt~-9eQZ?21u!TrRca_;-j0*i z;#$>Jv5=V-%yAGA&Bw=qOW&snnR}Lh9-Kt1;cg|)n7ScR4vAVx&DudBtR1pciyX(Qb zg-}T2Ph`bn0D2prxO!ny&f_g_4m)YE!C(#k0XI7g>~mMU>d;V`Ro)KUp3dsek&fF= zJG&a!6=}HDoPaZ(9>0WM4(cP(bnk%Cf{i>?_Y-R7ZJVD*#;kyRiFeI2&4fl2Ey{1l z0bP6D)PWA&@@dJzOK?Zl*=q%KKRa;g|3uyY$r_yl1nKpn2s%>!iIjzfT-7Ibo43QP zWwiEf1vLLA`jUwgY=+gB!8g=Qc}mil9R|HD=`hn6@A^yHV0K;Z`Usw8()@FEFG?MH znEME1(HoMHmRp`FbwNz1qBt%%=)YX# zAStrH+1UBk-~eV+aeVMe8QcHGhIW=pLl?yaA7g+QJAxPgw@QVlv3bAYyPv-kXlfs` zxdUf5Q+mB-o!yP}volrQI$8KK3FV!44Rdf4P?CTTF!7oCOVy<3lpMy!p?9kZG87705u1u2j(LIA6yBlQd({Bzl zO-?^fKc*Rj;1hAQ#oa-2+k3v%4e)#e7)jWTtTI;l+w3}(Jhu>!zo-}M~N`2 z1+y4yUQgb1&(+1^bOAGsIfWFWl0WZcJ+6^Lylq9d%Z3W!Po_z~MiLypO8sRYbR=qc zqO1C#P{0^ybOiCy$nKX>X}~E|#t@3l+>Xp?HFQoIdg1*!P>MgoP;!D?7WV9^nb}#( zIGz3`+fh8jATQi+%vH8+hW#DC{VHSXJMidNVaM_gO!`%h;P)Y4WEyb(FgS(2mlaZa zTu4-X)g5hitGBI_-b5&k(2jpZWFlnNq%rgy9mnd#MPJ^bbD+4tG5LM?0kPi*a-e}5 zNrS(YvG$K>JWDFn*sKD?9`ZHE*aLYtwwvo#PXgKiR{k2#4ln=zai*2Wb)T|;Wx89? z99vm60?zXCD!Zi}2g^l|R69hh0|Ss0qBk*%xVL0`(& zh;ezJPBeiN1lsRzJbpH1NOmU8nA_sH(14@e-;>4YSsD18U_Wq85#0vObqr-ey_8vt zB@=T*2Js+si>t4upAM7AW_&y&1LU#0Unj+eE_+n-tP{wS`ZrkH8bf9z5 z*}Ub}*wNYaF|#?uBRGO2Ow?H^@})gk5uVRM+2J5p1V?yv^KmxfbG(!xf0$dQ_R-T~+_>E=&Xx(aj zk%t)>$|O#Rjwf->95t#d@=1xrwp@XEnAj(+4 zJtpmuc`I{N#@Qi;!XM4r1vfwzjNFs}o-wmijNDXzHb<()__NtxCtwU9CP&#=nkDawlMp+u!Y&})U@Z=fE7jv!vOh=;j9M;bKWeIW+ReILPDNaD_8sR-P!e61UfVW z4eJc9V_j>1BkAGF6$Haq_0D&yuY&EuIN05S72}!#!o>v}0huc&S;3y!s}C(FnK=U8 z`{kTi3vkyp0XG1!@XQwA#j%PuVVNc7E0ktmeZCPVtfFU-u!$ZgXH;C*aC~|Fl)PY$nFT(daWk+!J2CQ;W*HBY+-i><=)iQm?HF4xME8_JOZ8rmjm24HB%g!J=`Lpex{yK@Z}qwo`dhdFw@q zfXaKOGD`OQCn1?})J?TISO_fsmC4%MN0d+xpBp9AvdakG!h!|DsSGKZW`^auH0%yb z9CU_IWxb)ncjSyq<)e)zxA{(oy_sws;Vt#;k<3c2!pN*HMY)e{s7q`g%abHN7z_8H-L#-tE4FGLA6HWZ1MIqgZ~G(S~>E5|zVrLDobb zB5TTZ7$+&uKlMt7v7IX#q-Ty~u5>K>KZAnYN+?t{GXt`LkKr)0F4yUsmZ*X|kLdLs zD%KB}KyI4`lNyTuKa>YQyiTLd-41DE?tKd^hG#nQm^f1C(vL{qIWK}0@`X0JvIEeF z*>HHoglx+MepC3}!EX(}r}}<<+ceds0Jb%pZvYc#7 z1HWHS$+o=2@3yJg7LlaS_TAATzIe>Tc`}#1*Z?IM-zZeye~-54^Kk7DOAMU3I~7sZ zs1w<7R$Tl|NeFKgsaG*KcFg-2P!U~Vlb31Ch0Y|h@3G3Zhut3AUd%gh;ap+9nj3}G zk~&w>aN&^h)y}KT^`g`f$BM?7JekGFIt2%mS;sm@~+f1j0mjo z4W|)j%3IMemx8#}g1C{$-4?d1(YpnCV8lQ7xW?XCgdAY+Vz3v3FUDN3S1H)D&Q|At z1T2abRFB?U#HemAQXBCl)g@E*!*s5K@+~v0n(`H&m;&Xy9e0UaXK9?w=OhYF)YU9gCC~O|hvsVmv?iCERu%lE9b!eGWSV z!V;L-jsS~eTC3MfHDD+>h^|$f?W~2CsCXSwW~bgDwM)+t5k3l0w`s;`k}RoQQWBBs z7qxP#G!LsE(JAcDQz2LgG!y&Tga7hH1X*08nli4EwU=;<9ddXmk6zhHMmnH#;Bo7hh~&GR}3PoZyLlo9Ahll+>u%1aps9(%$pM< z!=~%Jn7G9kLrYFVATO6OLfT|~Qq41PhQGk>9}u|C>UufGKg+JKM(EV;BR}?CXz_j1 zj&CvKz_1T+1h5a0t8$S2!(n=pFe#!}Ig1$f+WT?%BODjzOk=Jc7x`vZQK4_<8pnz& zi&N%hw!3oLv9I*Cf3b3KA#(BXH6pN^y-MmHF$4WRAM+u@+>RF)nFLxJ&uxmHE1@6x z3IJWG=Tc)_=2=(nNlw$*QL3mPqg1t_!+#~rkOXR0Lp;z%o2`yUufJjlXxxX&})_@&yAbP&?{n+vLs_&G9lgrLJg1X%c(6cT7$6F$n6lfY9#SS&`D#`FFbX7W_kGfO>Xu>V>WGckx zsUh>orn4f}!fU2?KT|&rw=-Ac6tO>PLSGYKx>k`} zHe>v}0yCNU^J|Pd6n?sa7WPEoy@0BBuEcqryMrwxwx8-rsw(;s-{j_xrdeV^tL#jQk9+QNIQT4{w z(H5*vaM9A&sG9#{YGzNZ8EZ+P^_Eaj3M^sIGM0kS1&@DvFRe^k00Kca-cT*fQ24u~ zEJE^BEMJTz2BTR}~LEQiqEBXdl=Tu;j?Gg*cyCo;c?Y)^)Fbg)^d+BvO7q5 zRYi*@6`^d{;u$Qm*{~-j^fsrAq?EL(nn3p1&bKc*-#Q|1!Yb&45FF_BoF7%bPx_jS zBYIdLV;jNBf+pRWr#=T$uuhmZwPs9iEGW?aoZ*)xZG7WcotxgnZISmU{dA@S+Nk3E+#EIz7I2?Qk(^gcmx1ic&j;iAPFJdksi z#3cQ~e00b|J@}BOqj8(=s-9-4@l`stPFA%D5t`|k?RTgY3g+0Nl7Z?^2iIy|G)GAp0|4`@og4`u3VYIjX{N@qqc6P@+*)zDlkkH0*EM zf!DfOJRIj2j@uU~X(>i&>0j8)nU)M(`X(uZbaj>?l&`)^=ZpGtCQB%kxdnvW)(&z@ZjhKo*bRP#Q($jI*ML6-<+=ly5{RDxl(=A*^nc!>I)1&R>=f) zM#>nm9-Wr8;yurjbweS0k%B%A#rpZQYuaE~Bh*3f-c9;&kQ7%}obX(N?n$V_AbD!? zP{)NHnj9`MUuvhtSQghrJ{H%ki^N6|Ak%~DM8>l>%ZQn1^y%&j=Z4 z*)iS`1P+||O`A6nD#h5XXz{*hpxv(t*~n~IrLW#vV}wsuGg&R3VL^bPQ znv5Z9ttc|by*Cbg`c83gyLGHCn`nh{RA9g0_Meby;F!~bj+(&|t-uQNO+OAtd* z?5kB7G4qQMwpFSGBBSo;QlTxyL)HBk$3Sa6vBfd--@f$Owms7<{{F4H_@~3EnvDeJ93vrz57A z+L+@DSyx%5EN1Q zze?pzra9{qW>kSOURZW*9QZBFe!Rmn#$Qf=hr3__CI9P82$+*B&G$}}Ie7!}XD*8j zT{2r`%PWVl&NJf5CQX>k*$e>k)U9SyXrc=#p)mmalU?Rf2_r7cRu}rb*jZF2sQuxO zXbI9d&?%QUnCMJYuRt6kE7}v!3kwsP^oCrk!XE!gnf??h2E?8{H0oCpAUhk5Juom> zio0Q>pNWloD5l25!9j30ziVvCAd}HmNMBX}g^;lLkS)!i4nx)o23F`%T}+T$I)Y2E^AHlHSi)VG?dxw7BMEo#ujuL^E+! zh|#(k>iyKr#A8O1?8GgL%%w@MZE*OWr_2Q|F1o-C0mJ43x1R;>au&D?A`6_;T;RN0 z{X;{^2)o%6@h18-3gpd>UnE&Eu{I-LtYiG_BJ6q>K_gsiSAx!HtxbXiPPE1nA85@yK-195C8Mfq`Y7o?qGwPhYP&e?31E-P> z^#%2jKy`@>P`Mz%nI{8DwL~LEG75}#N-)n^0E>q&Jqwbf17CM_ovNTNH3opB=vh+D zGq<0Z#-&h+ix}APn}phi;xcPX;*Hv8B?g;WC6}l?dn+HB-M;{PME`h+Qw=>LVvEn8 zx2R^^d#@~o!}B|1d{rrdIrj2XZCj^VO5&f93_=QeQllQe1h#bYBw4@MT75>9!{a4d z{Rs)GZ?wEJ95+cmnzLjY=u9(@EW4=uYyGrTsfs;Zq+{>Yy;{l#J_pXIcak&G*k za#PqT+{LPH)Se6u9E}#5)UHOct{12ZP>u_3{=%OkdWaItoP7}27pkRf(|t>Z zGx?V|IXW!D8U|#u49JOa;6UpU*^RIyt6RGY&!KRmPTd4(4Hd{j`A9yYR<1N*ag9)v zYc)lYffpTS4u(~zj!YS2TqP%KI1k3X@VP$|On3NG4|8SHtKGl`7T&E1Jfp#vG(;Rw<`Cwe-1e^J~OZ9^c3ptc) zar&#=jj<3N6KyD$Ht2=8 zH9?%c!AmfkkTYU7iESe{9V|@54I-BddCyb7gHl9>)I-k*LLyoK!<)>yEY_~$-aF|{ z$ZTAZic_Jm2PK$OlogMDrZEr9mM)l;8|8fdAI_Z2bvmKH#)n9FXTj>fA`&icm1m?? zd<@zn@!0l??n*T1<6Z08iO6hR9?MaOg<>R_S>^fxQx$2&HTbLQP%l@KBkH-W376*D z(;BUDp>Jk3Eh|EHe9^e~%Fi$oN>>cnB#)Cv^dl5!)SNuR(ZuSIO9z-hD-k$ny4zRY zflYKrf%={h6{O--dQ6Hks+>X*`+ABg#NBbJ8%50V&?N;b@eLp)Vy@_2w|P=AUt1TG zb3Kj#W}w2QrP9QGuC*hm1R`baRZt@h%WRM)YtABcB@9?K9`aIWxrma270hzqEN4pE z8BVj;f$QKZGbAC@Dm#~m;k>h$BXkZe2tZe(M|_ry_}}^IH6cbBB?+D@GfEOYm+QXs zj@3nT7LH>qvf627j8nbJF||mC|LiI0aP)l!b8Qwi{=^MyR#DisdWSWTl$lYzaAB5?sUg&Jo9G}9U3ay|RH zakB3c)ms!%#C>Cf|>5~8{D?`UC+vFk(*6mxVJP?hiCNZ_gt#@U=E!HhB9 zI*|>v9I9fw8t}I#W%OxVWqK>~L(fslMkB)CGS4Ucjp4Q{?eK@=!f#4z;<%67+sC)Y zZt>RA%~!jNQvzjvzIkI?=BoF75uYY+Ce=L1rDw39%OF1)mvY3O+do%y4^05^0i+l-w&f z=BcDl30hbqS|(XwCA0m1BZzT2Ge4A0m^GJi-Az&8fqgO>Swa>?1iD!T>aza<0npbG zh>KUeg*sBc)nA|mQTeUh45xlYK$m9Ww|t>l2z{g;>Jk2k8E({sws&rycW_6dhrRO+ zsflc7@Z@FewAJFE-hn#ofg&3=xl#2){ITbPLkTE9RmSLdJ@AgK5Ow2fUts^tC9hpN zh`sjCR2G--*MOP(7uVWN@~*c|uC+hq-L?5|F{m}-d_j9)h20+?D76!0TR!lZYs~;( z&=FXH=ea$uK&j)TC}b?2Z@D&qVsrWa#QPH0J+qDnRvhQKD8Uy@=b4+rpS#loE7ErIymd{rCyeB14nqKmkzzS@uBN?{`RrZVZ~5w$w|sWy?Fbc++pLbjiVh?EmHFCxO6J#G*7hFilPXfMjPgqtJV%p`WID29Be9Lp{{_RDbUqY}HpsM}2ETj;7~ zaI7|4JtqE6NtPOLe4BWl4+`9H{!Hh07r#+ExwRmEeXe!gH0%1@81h{=4a*>#uTJN_ zqceAK-3hCqtl~knE=KC`t4^p5CJcLQsUa#YV%K%XlE-1n+|O?}za9Lx^4r4iMSj2I zXH0#l#Xt4o7M_Zy;@Qd*Gp2{yc((C8&ht3WcAgmcJk-InBVdP(wOG(Lv`5fkgr(65 z{D$*O=f_^y$p(Idw7C1b+4@Kbb?ku=&=G|Vc&_XStj_3&bV6!ZC-nc~*e^RM_s7MQ z5k>Xxjt@@8yqd5g^@lxWb+-Co^?G?zzW+f1$j@WLV5h=@w=5@5x=GU z?&D|3Z|XzC!S!&SoIZXynJ2zUAI1ypLkT<+cshAHd1Bx6u!AQyPY>IB_Kk_7=g9AO z{I>9G(0NDtCf5cAy7h(rrnTMrfwlo-hV|?_Q_Wat)cTrT@R@7hV@QaMytL}mc+VXX zg~tZwHje4;nI$x)KnT*Jbmn9{zZ+$877|ptn4o)>Wbxl|O5i5^*PCrS}k4 zx{|Qc(xriy%P1xAhWx$G`z&1RjV#8WigtKt;0XdPB&EDkZc|QR-e2cE>^tc4T!u?3 z^t%0xA^xFIR_N*sPWIP}KG-D#Fn8Jv?@2TLJO*u~?pp7#Ss1FchP3UojcPMRpY=o5 z4b>^QoUMDF)4j*$2T!v?!br$29-qX0)V1*Zndz(J)it~r>FQE>*f*LYhypc#IiHx( zB4bg=szVo}whF(>T58Eylj2r3_Whgm?xszG?y;d55AyK*UJNqy;UDM`t5R61T{9h z6rHFvD~%)YMy{97L+QenYsASaz1STB=>ySpHlXF)%Yq&Ih6Iat^HmWkk7_JG)5de< zCD3J7fgYOC?WAc!1VeW3xwh={sRUb%>*ZuVi+z>WiMza8j0U^}>x|aE>&+OoHY@ zg)#S>G)|dyZi*BWM={pWMXySWTrXcq8WzfwTGtZS@|GR#m(T;%gp7r-yk#Ww&3KE89eLT?< zt35*zv-a0QS3deyRC$Bwwi{$l7!f4Va zZuNW^`mILV`s(0x?91>7CKxp8_(@Gk!l5zKi&L;4_W}RQ}VJdgImgOb+ZubM1>WV_S9*nZz+%Gd_DM9OM61$yBftXk;&a zK8C}e^)deJrIce0_3d`tt#;AEYP*;+`dVd^6PVeiJ&)Q`@Yqy4LL9Q=1V#z>A? z=gcwYt8%OobylR8%vRHX4t1#JT&+&{svPh6yBzho(IniD%vt3-MUU*J6{r|Mg_80%8%+|Hm3+a+U_CgcG4|!}eeyBc? z+RJvoP}MyntVtm*%mnCmbfsA2BeGpXcsG%)=ui=MEo?!_HCt^C~uk1{k=h*t0a=n$Y@VK0r54+Y)p+x)$x+*5>msHoo zSMq|OX++tnbnGVjE)J!055|2S$E0~4d$A*QxHw^PsIM=a;7ta?i)xcd`=oi^KI>=6 z9*&~uY(&r}Idaa?2{}W8Wqg@bmDA|1m$yl6h0nBm`e;vADm`@mXqk_msnjlz{nVuM zT(4KHwE0wuuX0;v-SV-BbA8(#$c_hlV>8`tD>xF7;M!2<4ebjJ=-%Pu_LY?=Sy4Hu z^v$JT=!gVBWPV0OlQ_Ayz#N3gTSX+dTA>!9S~4z)&#I!Rrl>@?AGT! zA!V0ON-Y&D-Kz=1gwN7e3W%fpl)Skrka!}*Bg$2QKF;!j8G3@(^*5xt$_|#f*5)%{ znvSRs_tAN$tlpTWX5Zvi%xNsHwcjRTpnp5c(#<1C5`2#D&d3{jMyhX+!Oc=XoQCjv zVruFThu4-J8uvk!2(K)mHwhZx@V4GFd)j9SS!TA;(V_9PVeKqG;P1cpnc7M{s!iJ<4MR9EF^@h7LI4lU!;Gi2Xb zOG~b$;%yHIlFOPb!J{N@I#lsCY&B&xH3uj1IMCRnf>+9SmAU|^rsj%muC-6>Q8oW*H&> z^mufwO(a3cf@qMz%aO^XtRLpG=NiKvXiPi1lZLC=kyI{m-4wf2N1SGZv}^kd7Id=S z8zzSlUac-!;ySi{@>O<=rL-S{MQnZ#h>0_PJ5E)PZA>RIam_rbCoeJ>phk z*gO^J7eGhk>FVxbTI%eJlZ%>VA${Z?h7c~y8@2QqdNdqeC!+Mt|JS)nJh@huqTuIr zk*Nv&StSJSfT&;auWT->@9Mimn*82o6w)8EKFMMy0(9@v0`(21*1xdS{?qEwB$e1z z%M%q;Pc&RCch_=g%=7h5t%W0^wOIA6)T7=%b3bl!yw1#p4(}kHBs5@MzbXEOjb*3$ zdbc7uj%s}(>g5;j)~jy|2LkX5DXlYC-#l!==GD13UJN;?bH50FNQt6Q9$U;ur91-e zjYgWWJo)y_@$%^z#GMY7yrK_Q%T2d)aB;fm7T;NmCp|C)`;!+w#2}&4GsKvZGCBRK zDFw*>r=$`vC_mf~>W`B<%ufFNfs7M``o@$*RX_s#E8c83?spi=kX)rMz{9Gq!D8H& z;M-~s-=@`15bfFde?XMI1IGVA$UGk1=t`l5c-Fc9z<8b zn+-Qi<0|v=v~JNKhZ%3u24XlhM&!WgNrOAdQ}hnmw#qzp7a9bAoahujLw{i-@Tr-QUHF?C`QY;# zkx$v{PHsekWbvF=Bv17$sF!B-)!9qXd^O{H7}K|uPq7qj<&R=LGi`UK6kYpiwnaVI z%z&_`Rtmrt0f3ofZAcgMH;y(AE0Q;LT_j^vz{M=g;Uo!-T$w|V<`#b~3B{0s@e@v3 zP22FEr+!o>YQopcF1?96XwW5&!DE@E*vdw$QETBC^qF_FZ^@X4YM={hA{_bMw9w~E^z zAi%;O^&1@!B^W7GT#PBam)6DVcJfW@+obg}Fy$I5eY2h%YTBzYrszv;cO?T)V5-Sc zcXD?Tqu6`8$N@h`-K5d>hQq*V|=ILR>Zy%M?S}X zxp+}pWld1>EOr#7NO#`O^pX8kr#aAIwSlQ?s_mD<>LDe)vrxUn-pB0jJDu+qs>wf< z-p3{)ojaS&_p_cX@rXU3^4OUSDMkG`^9xhPJQynjGCLP`8r>W+7+EgD%Ug z^SVe=QMt$cvH>T*SY&H8z9pO{93a=($;maTmRJR330|F$C%qJD0k^kL zp~}?tPeuRVNZZ8XLP>Y_swEyw>Q1fu^8#N}%uzqd_|JDnGQO~6Kc#j&OsN-^ba0)* zLI#KGPq0Af6@JXIWt^V<4w(4XnAU|lS^p|xT`Q6dYfE;M}g#%M`3(LNpmASD=e zY^B`$RXXj!c1cpvjHjSwy1&KzNv4fhTonD}?i@=G)0cIEfj=$*17bSeT@HY$NaK!9 z4rlXf%0gNoOL#X6ll_INbq|fh);enGwH(p^e}k17qGoheCrzsJPM9jA!Qfg%tWw0* zrZ7Ag-)$U|>1JH4Ih0y8h&F2ZfVZVk{aWfAoeYe}=Y5!bfBIImwKOzzxA;88jDay) zDBYM)pjdv`SD7>;`r9!A{Na%nM!Lr5T!Rn_(=qSUkZ>zV|o!m?u+r&hoijoXL&RsTy0@QlUw z5}SXq9c>bq53q}*D*m;{C8J8$y5E5o?)G^{Y+5u;fv$@0lD6!$ z)up{L8m~!dhW$06qlSjaSYS&fRoG*_ zoZlF3Lm*ULM8_It?b{6978#6oZG)2um}~7TWSx1F^>WwRKk$|xcvLDor%25D0*`jc zd-!T=IO#D~blGmMmzo3wU-`2mWsJeR4=f7)?$Y&?9Iz4j!6Lo_0m*|QF%IjIh8r`t zFTZTPby9!tV9w5E)-O*EWOJWtU-*;y5SBTWuGh;p)=jh6>a*$Boq@HIn5!}ER|c0! z)_;$m6H~4BG7}q~^9)`2P<+|MzFzCJoXkDTK3}hU1P&<0s0)6LJWTwTd0W+k_0d72 zX^0sClM|LnrI9VD?r&ypoC*F-!ha^nPFw1*Ew#hR+kYlFzeQKVRq;9iT)h!N%fOR> zYfQ;1Q{}zZcvLzNZx6Owqdx6AarC5*4p;ft%J5o|M3mWAf)Q(O))WL;$4}-sEh+47 zZ|cPNrV|l-W7*7HUuG3)E`#J z5&+vPHPq)#4pK6r#Uq)GEAf1OZoAbEv0L3G9b4cm|48!-*J#UCd@6Y^cde@=Dr&jf zp2y5)+w)fu`=JEe>Wo;cTDhovA$_$7)%&0VwUnbd@Gh>kKahO!RVq#p`b%xZU=?v{ z$c6_D$*e#%Vm$)=UDl+#R2F0!+|K1J08-b%(Ai>BQ?7o1LiT02l#t zcQuFhx(qa-tarZIiRBp-3j=oxiLfdC!Mo8u?G!w!>ophL)7m>?QCd%~+8*w8Nhd3d z<2Te`+ahY9S53<8vMWtihqrc?yVC<2Yt_eq))emVC83mREi5 zxxf08PTU9dhoSrwB5V)nhz-DY7$LiHmf^KY{INjH$vNQOlBeF*t#!&OU-5M* zE?id9ywcn6LagdfzzOaS^$~v1IC&xaC~pJpa|nhZpPV4uqGlPgAFouoJaHE_9V`H{ z5K)sD*859d^p|W5C#A|3x52pKr7c|N!ELne`n)+9okW+^$m$cm+A$}b^mJrdS;cA{ z_Pn%3h1Sy_x4*QJWX*Yodw1Bgc8hxJxO9?J<5Sf9*8|Ol-_L^+aleyw=_1~Xo=3(o z?nFVry(3Spgu3fh$urp3KE&JCJk$2*GA$;-Z~P8%DsjT=czq*@crTv1h)rE!?M*Z) zPyO(9o!bABeAcU4Ch?w&Ybkdl7uWuJowPAgGl~f4DrP~>!(J3c z+)Hlwbo+&->?T*8T1WUgm)^Cn5+U?S;P2`RevaVc1f0S=mZ!G7#zgoDau#u$(I_1F zes%BXx>$NRRbV3R7r{|pVV>Mow?CZpoOEb=@P2UMyTf5AEVA>aPRPhNXWYekY8ZKQ z$0L8k-W{qxPmr%JmBMB=`W?Yik`$%Obx`yy^R5x~lldBY4r*8=Y!RqyHb=GJ{k)8XN?Obj6<5VycqvrxQ$h}1ZR}EU;Ts((EX{0B%}GZ zwoL+=*|vNhkvQ71bfuU;0)}6q8qRptHcFRv!zlxhYlD!L zg3Rh=pYzLvvl|cF$GY~l$+T$X@=MTH7N}AvjX5HQni)MS*#zwM*?yg`>F-@@lcx)z zeQ2K0FQ}EQSe<7;lhsR1-3Vd*DKcOffuPI{**l#U ziFo*!`~_N)m0NrVx#v@M+T6JX88J8VkHe#J^OPR-`~hGj_qGOA*F&XY@x-46eR=r})* z7$ocQ>JBpYr^A5UR-k*K8H(;JX+&>ZsE$CzxEMrj6|9N)rbzna_+04|;UF%RN>|P*^-5Pn{Hb zhwS=1e~i#>HQ_s;Y@a8kP~El(ykxqa%Myz(k^(h@>8Frd27Nh)mPGUudR2%X60R6k zAB zylC9px2V1^({*eo^5vgeNUW$N%3EcR(7j^fT4n7iRxQ06<>wj4*8QxVPc=(2;9ME68JL9a`fF4l_<0+@{=IMJKuwQtZBn0Kzv4o_MDYj=gmUTa;DO&40d zo?5FXwCr`6dT+>_3k@h9@VbnV#X9;=fwfX(o?gVlNzcoAO?P`PEk7fR)yfMa@mx$a zn*&hMBE5xySz+5-SUecrTb}Te+45TJ@@$-MR28S#YV&5R-yRXkYx=P$E(aG?ux`?X zvkW_PBy7>hk~=i?Vy-Sh)+<=+I%$Ouf-{1aZZ{8!gi#zO70E$;x%Qg5sp^(g&ed6Q$%$^z84rzd_f#bzn5s`>fF7XS3b zs$fF-LCM*ul2N7h=gN3@5W`L}YX86p|5xJARLqDPsEI z6zc%HDj{T!7(RHu+iCicL6{f_T)TX1lWMHu_t6L2noc)uYdY3xt?+YTHj5EOp#QBP zBlra0D<_EZz^Lyu2=HDa78I7U1C1VgQz%s3bSU(?I=Y$hH0%YjLhv5-;X(}OaXmEo zon=hyS%#WI%A2FcBq%=q1ALx>i~4u*`AgxzOng2-je6p9EFoXUXOIQT#AjL$e0E_o zk5rpw@eTA}D%hOK_sWT1#^$9_Z0;Ld+>&eYR5mq-wy0l_`#*=zPl4=KiaNIG{~10n z?fCcc`G03R@p%FYinE3yVR_{UbVCz5~e3gES5|2oAULy>IOQ2&%U=eXfDq z4>UD}LvO1qi2CP-+{&Jrso40z)i24c_uhHH?U+3q<0|$0{sJ=HGh@1E0)Yk7Jt+rbCzR&W zhY5{UVtWNtaZ<5Etzj|6+A}m=CoE1Faj-`iY<4fAT<6H(D!+{)6!? zwf%SL?dKpYGQJ1B^yTsW+KbWgeS?t8o@1O@-d5^aq$j&KyXi9;ccgQ??g1fSK^qbI%iE+Hnp_-cH%_Q2-t zPyZ}FQ+k%AE+yrEgwGe?a{t}&IhV8%d@iFNJ@Gk$klyL<*)P%G8-Uz=fu_HeLVsuQ z-3$Fa3;q4WAG;9sk6EabKz7SiS@`Z`p(2%jjj*1TE+Rx%TBej|b|+()Lb%zbBb!cl z4(>+?-HcR$-fxhKZ!n>-FTLEHMx)KSaY1N@`g|ibzo_S$Y|gHYJ!V&D=GXMhTwO~B z(FXhn^T+=m^!NC#{(gYk^hDiQLd^cYsV6OJI;#77B&9}KC%$$Klo9(c*t`CM1l{djQ+{*~d)F)-(aqk~gS{&v zyr{h^mL0v>yS5_nIhVa_B_n`E?;Q57ai%%#Xw#Hd+q))E2f0$To4xCS*gOKY&1*U@ zoKn4Mgn$7OwRP1iI?__CRlP%1^KUwZdet|xDjo__3oIBASn$xxy{e1TE3m=B%7Wi> zx&O9rK-gUF--wmver@HNt9m?Sxumh32!~5XHy!$5TdzP;b!mC=pwQ~cHpB-d7sp!S5LuYDef%8x zhiW1K_W@8?01i~CX7dt*?;ujq*cFV#j$tqeyLeWZH$pVRbPb26Qbe@$fht9|Pg_&J z#M|}Y*VgD2|6iEzGfB{WzAx2tMJ&vj>oo8+t+(I-n_?J zep;;On+~n!xB&BbOg%H4a`sSzn__R(gPdhs?6YR{E>3Npr3o~)CduP4d%vMY1D zYbx1zQl^=K-Vjx)CO1bu$`eNJY1T3uV-w^fH;qw(TqF?{s*9L&bS`(f*PxAP0W>Z}vu_3aTc zMqLL5Y5H^ZGo7|na=S@j=5Dz$Bted=9aFiuN+stQ|46Y5RNwk#C-HS7+78kcC_{Jq z?l+H|?dHf?oE$mJ_nYBzym6ge z&g427lZ6(+)3@mmfwTQ1Lo-jrlzURjzA^SdY#S@G9NsywaEE&DIgYiscbEsSLSs(G zh(-w8%)26r+v_y>(wQnCP z*LhOP+@b+d5D{%6-!$er#VFg z;6fXZX?(0Jj1k|brb}%dRh&}vyeL^7o`jM`+yNTL7_^L+?`aiLVZNz5Iv+C4l8br* z{m~F?5!)~F_BCbsKb#{f7f}g~`?e?(OrH}NU@T`8VfUNQL})1(hw}CR^ca;)F6_)C zzEJ*!ULg=;#?+sq=m^!2szBQ`zl?gB%dHDkKO$s!>*E>Esyospd_OX5|I7wsdXazT zZn2({o8YQh6I5+Rr9K zz6KdJZLTV-7j^L2WR?Ci+RPL>k+n3&d(nwlk;PlND`qON99{)7`T7WGtOn)Uvqrzw zPhK2_^)EP^5Y@3objf`(MX|Q9nX`D!ChlDnQBNR@%NJKhM8VL8E^9tEd4% z9~$MVV0R{heM9`#w{n7xy0|J(`C0IL{NQeFESB#IYA5WN5dGk+T z*tNbQwj_D8^Zheh)pX(M#qf7ARl;WL2|*ghV6|Gi^gBMgv`gU|BYG0f&R`gp^gB`D zK%6RV?)GONLtW(CiDi(#|C78%ILO)ELERnWwx#tKz+%g=Sb?(5Y%fZ%Zo zG*}~;cB}k+QPm{mhnG5|M~g6;>pEKGBw0{8K#mp-Tq5JY1HRki4SU``AzP0X7J z%LKUP4hm?Kn=NKg(WohPhqfLL4yP$F)5*;EG+hf*naIV8W=(z!?x=<VyMsPUNc#sYKHu(<(t{Q=rOGxC`a+ z3`XBopf|0YXb^)??zT5iG{Bn6G^Aq+XweY0kJ)7E@?W0`hH8UnAy~z!`e0O^`rvWt zLnlNBRPM?*lDV9Tser9lY2Vf7m7mEf42PEuERLU_S+#mtDW{p=y*T@k6$91UXs%@V zJ2J#%tEg<;K`R$I45N;|%G_5r5RBlm`1ZwtrD6Zfw%9ZY@T$j8o+{=Fd#<53Y9But zE_wt+aVK5ve;D)pulhxnt+?lO@No#2OGN zzYE@!uQt9iNIq4NEd zq2<*>>x5@##v*&^pw;$_3zj?8^k~Y^w&M7C8M*e=eM>(r|3>(ptVdRUq~_8Dp=jzI zD#zS+t^F$l2OWH#`Xm~%ONVrD9w!>2eH$zi@9+Co-_1ZhRd(EFS#di`wkbQN% zB&uKg-s&_VS9lAamTapkj(>d7$+OlQ8KueA8yU*MFTQ{!c(3$xX5qHwhXL|c#uFa} zl>-okEU%?IeU&5YrX|QKjtxzq{JUV#I&FCP#>CJoCaZPv40B=gwNCF^&3$)l!H#ef zmr-$sqhu>1@O))~ctwo1KAO(5RUhux5G-Ufz}1oYj~56=eU-2J7d9B&;kw(mGtA!X z2cwB!)^D;{Qn`MRtbsA|xDQ^h-mZ z)zYupJ0P$4b~?J9!Nk`{$+EmkC==6M=%=|7H<{5mZ{A^D(-$6}9DH-@8KS$5GqCK& zTg4XN2beDih3%H^heM_37$fej43OIkI1Bi=tsb|}I|lFN4qvO~Qfh}M`hvn?d)4g% z?A>8&8@v}Twn(}q?A;yqHmK9Y*xCvQ@3na=i<1X0tS>+GA>7`u4;dUENUDWf8Hw8q zizgXVWLyp2j=jj65<#djMaCy_I`j_`TF*eYQ4w%EQxM5bf1dBEMaeSlF*mmIqG#um z43U6)Q=U368nT`c!@cQotGhv%;DK{@2i%RAe4Q~TwD$&%QZvNjSK;@#ea+>Dd&`$W z(*WhJ9}?@7JK7^D2N`6^MKp84XmOwU$9@?~P|y4%I=uAA>O9pjnu`F1G15G}x)0kB zF;_9q?Yfx-5mz@om@QTncSm`ZrL5{QA@V56ZI5zqP;Yk!*5F-tg(YB5sExPup^oz?gH7^+kyg4+E(&|~hZ5a& zPuT3)CMiFr7Tlywn)`L0{I=$rtQj|GjgTYX1>s#{hzF_nER*f4FkToyg?F)}!zwHa zI49V)%~p3Wphw#00|j&6kyj{Sj(~F-`x6cogm(dYIjkGi(HRh|!#L;09k2&4Kzc?s z6RDNa$7X52tu zs~u~Yc}pVxM?AYUaEh;`4L1@ur6zyKO&cHHm72g|TO32CLI6CIJ_xRTZ`B1&!&41s zYWWdHT3NZAE-#hy^*5y^`rfmIE|0{T;f-eaUP~xZ08Y{u;ZpCWrUj|V@{4TR<*1b0 z&OKJKe+UhK1&X6T+vKzVEAS_svEpw+>wU2^DJymPZXoF~*Ugo_Q!(CTvPg0~BGT?F zs#o;YUKQ130Qi2><%66^HshPgm5^H318hThy9WXK@*^VQ+lhltt7oEQV==SYzVh-V zu0n2EyJEZNBGYe*_-uJ0!7|S~XtUH)?dQ+9O6*uHWytFaPXPOltKwH?okvts82OE^ z*M;6aHbv)qPuo#f^${tXOWXG0GdHwbibuA1&%q_GGk@i|7kT1-p1hwS?4Hxe7(14@ zmf%D8wrwP8TH^ZfT@rHh+u_fE2@>`WVV@I*v~f){!oW47$b>|P-zNZ}-SWK%4cENa z5088%a#E-zAtK`ZK%$P;Y)1HKQSv9+CayUXvnj+)slL+=kE7xN7j5v4X&Rf(6A@w^ z;zr}T zKWm!tR}sPYMx9g!-d+hI_gL^G&=>ZlG#NMBlTV z+^a;6;wq>U8H1dsHEIfcio*g2hFC1qT@`PdkB802H+UfCBm8$syvb!GXHr}bJj1xa z^LUJ_qD)%xayxg?rnP(U^}}hhr>X+ZnUosHS&QDbaoYImoR`J`Uq$`2q;96#mUG4^@Mmcs0#En?b3iUOMizPm` z_X;Uk7OITj_?)=jA!;?mtau>e&t0LXfs#w6a~#nMIeM$mDzmsgWY(wm;tEZOzCiTt z9b&v$p?*O0J9mX#xoyu1^_XjcZlaS#4HyOu5(vt z4N%;3otcvm9Z0;rye6TyTFo)b97v7>doMHHgct5FgL(g5ILS3Qc%$($Cg_3)#P`X*% zd1Q88@5K!;A0`UnDpY#s#l!ln~Hz59Y?z+pt zKOhEQ7s6TxLRi3dE^!m=K46IMh4Dy&7WHVIU5lyIR-lYPDtNv~{mmTklz3U>16WERXaK z(E)cs#g77Uw0DTT8e-NlAda28S}y`+Kre~By}agl?|FXTEcJ8p{JeJv zu~8uX2_R1N4v}Nl>m(2-&s{HsK-8-}GY7d)d334HY41JD!x~`CDY87(JH*>2#AzT- z_YUz}4KeEs5NFO^t+hZIxgD7~9UiYa4Litg(3jZ<@2%kA3$ps6*Vr6l&k}pK*Vt6( z7+$e+kjM7UGI(JejH<=|A8X$NA60ejJ(-y#Lo&%3AYg!iQG!x|77b|P5KID@M3gWg zFiGU4El$&sdJ)b5D&*nGXeP&L?QQkyty&e^(pFpA>jP0BLm&@8d1y5eqTwO#@gRnW zNqCs~{%h}@z!`*7TrW~w zg;VHwibBCD1_kR=NV}>{v*v1%U}*z`^*Ry*EVwSs9fVV;BE?lW#V>e@Lcu8p1#1=u z>l*37YWw6fx)U2 zm4Z>^4#Fswh!j_06y#+Yv?UnDpkR&QV0Bw_yZhJ^7)M3-z;JyD1u$YsgdT)b>=r4m z!YMZL6orCQ3<}ow?$09|TV+@eOEWi?6V^FY8 zLV7?ggjjG7speyRU%BWAG$2T|9kb>hi-i2xz;Hdo%d0q!GKsYe0GBLMgxSjgDdzJO zg@VNl3i)IX)=6vb$w;sUkxzmbc|QyJWI(XmMT)DinRj`LLcwMR1?y>~Qf*&ZbH9oN zYY_RQTvRILlL5i9i4<31Gid@W!Da>ptM>wdb=sPHIufiw(-Qe1`2bU||rS{H0)P_Palm1^s< z=EBkTpD>FrktMHW+o}y6jl|e!6f?=s@`@)+0MI;6XS@*OcF@V0x zy64X##Z~xkMd}UCumT|BG*4#iOSc9y4ZV{DUW!+;DDXzj-PJheCf#55Hg4K*v zs_koQ?$?oE4YKa}JukB2D(jwyMT)ENl@bA#;46cIbu$O+j5YU6A8~=tR1C84F^F2P zvhX>2p7--r7|RZxqEIlFLE(B1sZ`sy*4%F+(KX1rXN9QrD(jv(B86D@3^;%%3b2Iu zGALMj4%P*0?uAIO23hy~`y9u`Rn|TKBT`(2ul$CmC=`5UP_P~l46 z5|(wv7~)qhoB|mL!k!zHvbdyZQYQqCdYH#SV`}6YQ?eGIJt66>hlE~sz!dvM3Za)B zFa=q`N^jj6*2^YdDI%%<0C}2GVHDtoV>@UsV_)f$3 zFAnDD{_yV{ou@fR_vF4D-N6GnIupLXZpzVpyEjL7JHA(b1n1ZIUW)IH`*U<{ALi(G zH0J1<_T=cy_|C=mM0`)hxBG(}-QQ5hZ+7SC{(?F_Lmk8M&A0E8VS7dfitNZU0Cfpn zt$?Na>U&}F!*3-@g0-I=uJo9032=x&m=q_d)ggQ0Qe4M?ZN712!KGF|%lpBM`#$Lk zVSY1F;A4bcVc(@T@LNS-Fs7`7)$$az0TB3mOz>t=1h<2*UDk_ZA+UAKGr9D&d6#RK zeAk%Q3zs1y*iY9Hk>xb$ZbG@WOD;F&eTU6!=3CfR@n>Qg5 z!Osf$L4q{cZjH#ugKA_g?-veZ-~qhDW70IY#Eyv?c9WKYB|O6O>Q^X15a~=?;w0z zKmR_)amCuN77dK?sD6z9ZUBrQ8UW+l5jHr+j{*eR@E#E3oooJ|FkTXY?GE%H&Rz6j z`_Tw&|HuCl+sg(<_l^*{Q~S~VbO_zTQe6DfX}w}Ja6a~wP3ed07N?oJcS8M$+ZO8L zU^0HoUU`+mqU2jdsQW908CMM}Q7`$An{ams9u~vr+L&c97n2jiIkJ(%eRfd;rTs#U zOW(UA*xOrSxS*T~rtepF8mocVsA{=GopTxGkH1BeN5KaMUd?#bYt75xcym#MFSmCY z{eH)PFXI;=gRK4^?Az5KH*7bIU9!Q0|q8+ z65YetXm{8QgI!L;@wr?TcfS1VRbT{%yy9U&@hQ5!tW290TxD_vVW@y1Wud{;30sJ- z_z4~ug`l8=a=4%^=-En`x7YX54Nyx_*W4Z&MIl3rK8yZ)i2}%?A3$P+tGeoppRN97 zhkJsu&SHSMPIgjOYV>bM$z!svqeoKK_4G){(q%svCC9*}`%WXA!b9-iSN`-7_=o@J zbQmh*qBz1C`vb;3+1^Ptc^FozV=+u^>hx_Y%@x!c@iTdp81@Hl>|GZ9@6H{-VD)%euh)tSEu z2h>o&^6Jdrh66}XsTEge`oaOU?$L6qGk+Zppdz)Kt26&I96)A0aBr0fmAI${d3Bmp zo%sd@!1M5OfTatrS?)3g50!Z1KZYA*J5U+5LW_QIW1%yuNKn%^vM92o5oAk7wY%mmcG~`W}&2E z6ZaB847>JB%7!rXuuiR50VA97C9HWa03~HLKL=eVmR5 z;;|j5X|k0Q5|uw~Aufsb>C;gkw}G~mtNVdyQy8x^DS??Cy#)!#dKX5vwiVzFrk4qw zcn_{w!Pd<|Y&ZvY@OA7(K+^yK1o{%FSKy138Tkvl2iRgeP$w0{eufMf;pKOHhVee< z$2#3cLR}d-X+7=;%<(UdE{zjb2T-ngMedd`=qtIGwUZ)U-aDAcWsh8$69>CJ+o8QeD|4 z>;)p^F!++^cfga%qv?i+OK?)^>RqJf!N9M>dO`B-rs!MfaK<9*Ik}`x2WRI+n@sLR zM{q9WxV%(32u9h%}5*BwN0?5Re6_dytmgsdx&_(SAr`>5S0IzT%S zXcGA{{74BJBp*Kx)=h9u=luhJb2EeTu*`9CqO{#+G*2817a)j`fmiqvKH*KAfaAd+I=$P(OU%~d={zy6Swr!r z@8RP8Nnu=QqeW^Y@Q&k`0HDAa)wso5YRL~|p&cN?kX05-hrLAu$aB+FbSKS|MMX;1 z4D>SW=2-?tCeER7hh;l99fnu;!zIPFV_!Q0${#Uf(>(y>D zCZ`j~#jsb!$^iDNmvP9e8fxF*9jo(b>c0?No#jh<&U$dwt@GLMF_eW#BOGyxoIc|p z>^KjdN}2O0bAdN26?JS(!4ED=F<@wfOcc4KkOU*4WV%SDL>Z6srIscVoRJj*jSO#r zoUG01EpQiEHS>D+P`(KKUA;|h-h4g6wCOlMKwzeK;V{>=%7$djH$EF2uwRdxk~_78 z^!oHenR&^$z0jMNd@!KJ--+6djj@aK0cVT*OkE5B%*mXa>^7nDj2T1ZX*lwFle8U8 zVi&1NL3ubKhPT%iH1MYeyLZ`uub{yeAW2*lszafHDfk4;z&HvrDAxnCfS6pfg()Sm z0A~5>^b@$T4c9@FGXP><&!DX{vgD|I2kXLhwze{I?~3btHD7*|3n$~sz!PEgnl%x< z3RWDH_lP2jfhl>WgZYuEYGziMuQW=&o^F<&hz#CX&b6~0I5L(j7yu@M0;S1lAPkek zRRY7oe(Z!WIf34F@D80`-xQqkUOd?0~B((A3}-K`{@#APcz z*Q`22mkz1M-A53_W!&mhBy02=-E$Jgg+qjIGV+yb9!7jIYJLt+0z;~i@P>d|3IHex zAyA|ml0&O|2_D=9huHFUFW7YDNtmR|w*c7iJqqcENw;z{CBVv8SN;{^ik%zJenx}( z00#B;OK2rZ9#j4r3UKfQ?B)xcy--%FM+B}zU^bwdv1Ymm_&EYxhVmoG2g^4ZC9D1^ zRKdZXNu7=&b}uglM>QqSW-8@G7{cuYp@S{!>8phV-5bg+sZqM}P6CV_ z5HM*auJeMI@{=`p(kwF@mr3lcGtR56Nibn%cV?!{lZWI~UpS8zullms=P@F*cF85U zQl@x2bKDQ6Puxsg6-9m?JVO^41~^T#PTV79!S$lf%oul1=-F9} zkvFN^ez4c(fLH6_2s$K)rlH2$?94p{yrQ)X{xWZ(AqsnT%uL}_pjJ3t&4N`gL@dE@|y_I;qQe3OdPJN$bZedn3FvS9=4*^dm~JcfG$a83{!EAOn! z>MS#$K^aob*NF8Qvwhh{_3kH(KI0Q=vNx|2Yo15dkC}p*^`B9=Mim3DwgM?D%28}+ zO+8V`nn?4!d~G>BS=vK1_HdF(PUJ@`f+Hvxcj1vvbFP^=X`0T18d={3*HlO?<-F2stwAeUdgi*%&ewG&eyI(>j8sKOI98=_sX}1O4z#<12D)^ zshIEKWis>uYDpL`x3CoY&O0BP`TNipO~Dc9`*s<==XR_vktk)G6scp(}2`&`Ysh=vxD*allJk=>x!koD>$3j1;^g}^8+4N(l zpL^+NrO#W1_T1$@fNAkI^;5rQlqJmozXFJj^4)2<%iX0MljIaceTCD=FyBU~aSlR{ z8|0YUsGPgp7fTm4bRSk4ER#N*uYRZ;6XA2*7yVj)4wQ3GfbdAPHGv;=%jx#C5yVAC&K3;pNHWo1qb$o`3&??)c{JHdoZ>meO9oOIIen5?R9BQ{K5f z@+_1zq4SQQlvNM+IP&xEIVEc)Jw{~NV+=Yu5rKHbBG90NZ{;io@Ra>gUjCxR zgaS64k6l0A6`S7TbDt*;rN@bz7~QG-Ty!|~G1&G(n1$ffbDxY;{yDNp zB8m;C+7OE|GL`4OGGZJm)2!FKNaWsfdl7Ga8 zRbpt+z10uv)l0(d0d#Q|h>b(o_24?Lk($MqA74Ap+PpRi*T*2_`eoh>cFqFT)Cr)J zn1LQQVuBZ`4St3jYXL3yJL7&B;!eHWjTz0_EF0jb-37P*zNPx&`K+FDl5-`ne-kXT z1J5C#$~whuY>b&=Q#at8I3TW#flS-moYrAnS8I#%+4Qdaah~@bFm;l`heI6toxiuG z>S#xme`T$yG1~woMPMFEO0Taz4%}45H$V$KXuyVWu`d6F- zQ5tScuaEcRjH>%aN^H|BjRrN>sK!meUYahx*$pSj;ItZiA@uYDk6LV_O@A{b`oIR) z1hlGagwG#|4hB|{W=_@B>1v};1TO$=d+iYAcAfP2PNI|hlXUMuIHM(v7x#w*D~f^2 z`Nr{iN#hSf)DU+JOFLTFFTR5qHMPuX@JnykD?OQO^ajj0V7;c=7+j2`vrmDTx*7Z4C88DBOdsbazpI=B89IMut50U02+ukue-dukIwB;X_$uCJN`z!V}uww>k1wIJOy0t^tE^O~%bfu^ZE$KmZCr<9d(T`F;1a|^sr2UCv z`A8$FBJep;^M(|ioC>e+5Sk8QW7H16L77v4TSJ4L9RuS*Nl?J#VcmO9$(Z7fX*5jn zt&Z{>su)pci{gaR89|iLj%VbhQ1+dMPK;5yi#Qfn9W@&A%;hs)ghvbqYXOIW-U+ki z8egG4b3>ABXpG5I7bdH@=BkXT@)S_(oQGq&Kc$XxEnj5)aJ@<0`CRuU9mZ;0u!Jb=9+*3MCA1OHzYowP_He`r&u5@Yrfm-8)a3r95d;_C z+!GM1k{r<5n{g1{f$hAHio}uqZG43Q_JS>qdlu7G01Q`a;gGy03+HrkY!-r%OFIGA zKh`s}->iP1Z1{*UI;j~rJ`5bw$#u9snguv<%N~#PhoTJo4Sx7T`j&vFV%0fsEAei#YpVvG=> zB(dLPWVuEz@+PwiD3JrtffOLqLaC9lBv~Gfk|DqkR~IIM9B$|2;CD6|r%t6cIgx?j zsM(O?DlNFlgprcA3}eYwrs)lx7{PNW^c93+$_M@Qvn|eA02jE(4OCZ*+SC2HX6kOW z_N>>dJj_9@6Zo;PMwO8*! zuO+iOsD2^2uL$`Rw6G^d_$m=zEIJQ5kciF$O;EQ%^M|1Uw4mRLh|rtt?D)AW`U@i$ zj$^35sIz_!!4uQwQnAci#I`Mr=&Y$fAianN90-w}4QpZ1fZ^?JOyLBXtU=J|3nvzas5>ul zTJ3B8_62>-7n{?h`|jJE#vPe%Gx!_ZkKI3#0ycEk#i8mE>J#CK@AhUnHkSh3-`44K zp@f=-TdwYx<7ebixE=tWaj0kKRNX#Rz8K17s?*;d<&K(D7n5tNr4(IDi7_k#XTSo7 z%ml>XA%Md2k&wCIu!WYEMRa+FV9D|*?!{s@P)3twUXM45X4)L1Hk4mx^iDVemuP^? zMU-sfYX1JFip^0oJAs{03{Gi#jGeu36xiyyUo+caBw}ZKFm65dW$rUPvI>#VgRFBK zFnz(mp)T=?aRmiiVqG&di)i4vr5Smbzu!DCx3<$qlF5T(&Lh#-d05gwsv&2 zXwnSf9tw0(w%uo(iJjJ6Uvbi2x(s(#m&1P#ZcT=A3r?!oot#M#8`oofPXE{}%)0yF z!7BLH0;~qa1=rrk=^a;1E4>p}om@(_i5onyRr)1*Ug?RJ9wTCv-i~qoWSML6BB!-M zj&|ZM-gVO3Hhtfh3tvY1x({*kVxKKWdfVj08nno1-78z1*8MUZYUo9vTT4IW%(TeU z5kk)?p|q(3rlk}y&P==BO;{0ojxdk`s372n9O#}5=E6fzhms0XFtJ}_YNrnFWb+V3 ztd;I8(bANR>$!)$G3+*k6G@rjFd~C3LwejSge=~uWx}aX^{I8B{(csKdB=nvib#)MkTmSR@ovg=5;8!27-rI?qVpsZ5^-+`Ap zl}GiryT?jz8z8k&J0CU5|F*M{$Ogf$OkwwG(d->Shk4QKsl|Bpd47jqdkL<4*Hu9~ z;(3w=F)oVpoJL2L-%Ztew>^bGb`;E&SFZRx6d>*$@pV>nF(#$vgVxos0Fa9c9By+%)yw<6VB zk*2Noc}@UXsDz@Szd8j>5oRwyNDRfz|czw{$Bv}qcA`dy^hIYvYiO5LE(Zae50)URzQor%rh&8!?|oHqC6<1;>S8s*i?Wdi|ZqWAxhM z)FyADx6q=^2z$BQ&3(6 zM^tetktNVq@Bj_KE2afZ7#k@aXNA?LCW4vtGb>Dq!4eomfEC`1t8kkkArJx(ZkR-Oma{RxFu?{X!6WN1Ov0_xDQvms}HY)vGZY! z1*g+%ACe7Af*s#q*$qY4?+b4RwV-y$c@3^>ST}ZLs5e~EW$*t8cLi?0#;B`z;C8Il z#5&j_pUVY$bvA=z!mn`CxhQAU6(&wPGB$c zxa$$89fN)kJAm?V?^0uz-y(0oIeohFE{5)*vzh6N@ufzkS&zHJ|6)b~?wigSAs!#j1ugXN2*Qg{=Vx%4Q9SquSLIOf=tnsV@#EC z&0cYF3A0sVy}uJEp3h%f{xDXuQSR}OKT5r60J+D!Gr}qV-Kg>w{ zm(#U}XxZy|mH)oMe~FM8gY74i3MpOXY{v-d+Ggx$%cGPF zy>gOzkWg~$TK};agSd!Xi{8pnlABg3d8&~&BC(&YFYk%=AC37w{?e-8LzI?0Pg6t4 z?97I*I=Gd<03~pmaoahbUJng0uV*uUYoGpx|8Bu=4RV~O;`sR_Zn?OxpflQ#k^`;rO@8G3f8Xn5Af?Csul5ElLTdd14ZC zp?gz&gvcOpeFYW;jyZNiSSP#&W*{qSR#>WRQ*)I8AD%}OcWQTbyn}KqH)5My?X6<)y13KKf(S%lQG=mR6qL5yO*aocDF`$e&9`n zKH#G?aRiP30E}KZSp=R&4567C75EiCsOU+eXddXP7hsBtB=1V9K;R&-oEJ-3BZ}oE z&EzHBDd$yPESEDx2~`(YKsj=8t>l@E*y_DxaubLJE~_$DxXsxJAhj!bxKs980;l;i zHE~>^9nuxL#p|2db*{W|D@+STqaM?QXJU8lMq8xK;+`}P=`XK!mnL=Yp0#P@ezgt z%~ctgTPQTh=ZoN*Tkyf&L4MUouwu@si^{3>P_UPW)L>vONBltfsu7MHfy?9>1AW!A zT>ACT%n6y3zrp;lbCgwO7Vspucc^CAL7Uxg^rhX4`jHb+LNeCB)y z{aLA8grjaaXS=PBK)V|(R9a0jUozk}>*-bCoJ}031ghGI1(cM2=uTP!;bK-jboA#Y zH6H8<%s>u>p+hzUC^U7Y!eXVG8&M6lIgMh&qd-(5PnF)z$JKp0q!;o(WYi;QsGZ%w zmrA71y0*zf+>ff(hS%j^t_{u5l#T)gx&pgNanwJ|kTG3dX~ddmva|z8APSa~o}09@ zzafCQ4wL%s1C=m+JsRs^9XRVz7I*+5O3zB^*T+$jRI?wySZPa7mhtePOTXeb zmiPQZUas^!C{Iv&{zLL)A+Cux0yXMt4nMpIm`D3WmuPeQP@q;`W@90H5$o229#l@5 zT@9&N$$DcoY>D+ZJn#r1Z*R6JL=+FUOYN${1Z&0<15&E_6(Ce4rO6~%0B&}zyd3yD z0^LKaF83z;W) z0DcU2rBr{3X6fKIdmE+KK{qA_GfX7cMWih&tJ+3W%g@~@5Vax_$yX+tIYshJ1XDpz zpnbN`=%9n`qAAosA_HwvWF%CS^T+BJtI?|P&$ z7BzZ40uWPcJ^#j|iO<{Ua9v<2mC$6wU9NnlqOKj<)sQX6NIUW`Yl%ernu!A9fG{`( zRWEAM>b%QpFNLL8dn=RI8?7I#9)@j)-tNx@@B()9_fzOpz1k~=GW^v=!%j;}QjT|T z>8Cd9feJkw2>Roj+%euaX~y%p=-2+0|W*cgg~O&_OMc-Ji{VjpAWtN5pSU#LNH zFBh8Ci^_$}iWr0MH3W$YYZD7q&XTmvDxXAsdEfyyAp-z9v|qjzR1;k_6WaO0Ag=(* zUhO)VV1cF3D8nKo_D*ck$kEwgA>U4F6uFgcG~B_?9tYkD6Vp*1UgaSI&r`I~T@6v; zuSpv2P}a*sMI1OPwGyd8nly(_+D2?b<)M>$x*E*xaaVBi5?3nI16Yv)!eH>tO~~Nm zGE9+}CBT!Ye<9Z+A%yL&29rD%7dI2ZMNB5pKeM+{Gf9-&h@qfRs$AZH+hQJ8KPV8$ zl-`a~{Cc}92DUO@A0Zj#zr@SGg)gAN^W1l2@*w%P5IB+bAYbrJkwHdvD(@nIrSig% z`5A2@l+i*A)5Z_fAP2jh1ByJ<08h)h@pKXO4^i_a5Q+APWGWZt(5WNr?14K-iY*iD zAskIJXr1MI(w--|3mxFE+V(tR`TYrTs_$96^F1tD4YAK=A_df`EOQnuKt`fcHU>jXTjH@K*SOh>o~;(8z(%;WRNs>eBZfZ98{Z0B(E3JLiKq} zUzp^TJ7CHP-_l=(WPYSgMmmcc16#+geAI4HDslH|9Q1V%_oJe;LgfY>8QXXcS7c)g z#10tlD#39&J0oYqv?;R79K;2`8Bh`{GjDPahtJGA7`JHTah}7Kdli@sMq?+<8^hNc z%7nC&`7rVUtS`TWEv)otvfYtm`BHpI!;;_xWA!Y;$jh(x#Uf5N1`jN02#&BIk~j3_ zO#?-n`l4Tph^DKS!v&c8Qf%%^A>%ITkOG|7!tZ7dj_ARPn5&BcO%ZH8YB1nET@O!q zISDOD?-G#XqHsCEL1*8B2b7*i+)*|!Pl%nZuBF)ZBBFqm?=A@a!4XNQT%DfESK8$r z2r3DFEQ%>VMPC~UBYi>Vc@UwYFG%+XN<~fY3o`Q{YHnZ9L>@%#>kG=@L2pC`-OYox zL;1YkwH)Kpk0wcn|V-kWKc5?+7}tr!GjJ&2A$_YA4LRhY{f^p>7_{I zq|w*LBN3^}XY-(r$e??9(22;PDjvilgMP<@PDKW7=0Tm2LA!WRPh`+XJm`F6&gD2Bq>KQ)JLI9%POTn$3eOkwK+AhzKxT{KjN_l$TR@IH3P!5a8hs&+F7*-d+Q;> z&}C$8BD)Q&P`FYbeHu)HJo%TXy^}7~Mb#RmoEjKH&vBny22b_zwIf*^CB9xe2)px7 zFwD~Hv30&if_ZOMCYVBY4&9)BKFrxn%8YMo8^Jx?!yOD3bzJRYat&pN;E%T~IE_aJ zN2~yds1(MNlvwR8R&j}se2s!zt>mFl?(af;F zf=f<;qFY5mrO{N~%cGCMAQMM=GU(h%(kIBAMJG}q*s)taL?gzrT0DfI@??Ui4Cmj{ z4`-%)M^LyIyOLQVBw%sZio}Ly(L4dAI#`UB3nQ~p;HYIdJP^*1T+Gj6IG7!}QSe#s zG+vMfHPb&ckTiU#2kUp@QmpagV6Xw@SP3uT#_#BXDvmd3Bz9wFDox3}(BQ4?A{?a9 zoY2?2w)FaCIFy4<0GAI=+f7ghN30BYH!;A;FyzK24GtS=KP77dXx2-hb0gCJK#N_R_loRlqAhn1KC7I${82<6M** zn(Kd}1%unsgO%DM{;)Q#8OEmc$4iLUvegae`3yS@#oOl>u{!`X$tIoJgy34EeZIY@ zJPP7@@a+^J(T5{Tth^kU_S*SI%Pb=R*b3c@iw2YFI9ZZ9gt=ef0 zVi1y;gB^qzugxUXeKdG+egw7HS*#{{L8_s%j3w-^@GZs_oo-`#P?Oky^KVjiI&o|) z7@Y(vKs8(~x6SkLGim(t$RW-`w_%4uaSqMYu&?rHYh7mu{WlaV6YjrVXB!& z)2=|GcWE+It<{BQJ(RnkYJb6~BAPpa<9Xj+0tsBV;j+uH)a&lBPG61# zG8>BdF&x#n8AzH9<)I>5>}(1xen~58#JQytN^jEoABB^lZQyRi1b9er+Gcj|bfo5A zO4??fO?xFgxL_m#T^(})h6a^`KpK301i?{~-o>{$!4}zNP%_!zVLRD9Av4+zl&l%+ zV9#TaRX37}Rhe}<4>VvK12**T-6~9~oktw!&JVw;D4pxdX zM1pWh-D|?7e1Or>5Z1A^?gX6rhGafT&@8@({c4oBo!e zG)rV@+b#^&yd8a|fti2`MzHI^vqEFVz(8PRny+TV(7J!&#Mlv1SXLZT=+-oX~2xH7dzOJtAs)rD;-5Kfp5NKv6SX%jJQ zKn4zW54JDM8d$P`Ft0BcyYWljHkN_bh@EuI0wpX51D%tjou3#~f>G`vUEDe-${nu+ z_3o6e?AcNBNNjbCL{e82c105?Pnoq*PAOqg^bV?h9#ji-#)*wD@C`iuF0RKwRh)N1wb(EHo4T;oA*bROUigD+Osw=4&N8|&%!p+W zzrcV6ouZSTrA|sYK%JzX^SWs`I(twmrYt&=*kR40(J1-e0;)noZ3{-^Jv1WSbGl(M zRs{x;9MRG=F7(@eVqy*Aqg20KfHs~3#T$S@%#h>&SpP@H!uW)68I7$(69 zmgHS(@{aQ^O;HLh!7v$ccF@Qc?I(7?QSd&sfI1qt5u_iwZXs^0A~uX6{VawwnYTvS zs%ECj7janaK6%QFRr2*S*2>qgVc`RzY1Md7kBQ+5C_h%yTIzzsL%D3U{S)9HZ?r zb_S~VS0O@^`AtCOeMP2fKULTQ*{-le*;*h^B3MQ#EJpG~V}^t#q#lHj30+V>(-o zbV_dZm%$yFAaBA)YHsy@9y2zSQbJ191Mep|a@YzZ6P)CWZxtBveA`QNG%j(FYF;2% zLfCd09ObV<2xgyP%oAI#mprvd>Pm%uaVb(|#K@O$`c3kzLMZ=oMVD_v4hQ=X9FJ$g z$ZU*8<*e)$z!EZ?r9(Fn z|7bEPj0;coiSo z=H4&f;4+al%HG}$< zdPenwM}0Gc#0tc=AU+s8iJ=N4xpec({{*xYvARW3q|u?Nctx6Neo;_POJZ+Q&jFB~ zCelPeM`FTUI)W{^xo^z9H~3IHOZ>gj>~Xv*9oLrmJ7NstP826RQq~~;xMENG!(w%PxvBUd9Q+_hZbE*`aXO!NG%JE8C5hog4nWqZ6_ct7B27}J+q_D`@iO)Y z5-fL_k?U4&EoNt9C-V>>dkE|N2^FB3k$m-Vc8tERCljyTpY!4ju92lQlVN7sLNn82 zFeA89e0UUDAUNMN_11SM6+&zo-niM-K|zXM5|>~Kn8oV zuYkb3SUQ&~)lS9J8cm{=q@>{;D*aYS70U5z{l{bYjgd#xBraam%^q#sgfp|f7~Rzv zFu}?-*RU8-a5^uT5E3|pXz~m5A)cG`l4mcTxFiWZ{3#f~bs5?r6qKf~KhRT~HypEB zfjqLn9j`WaAAlQ!!^+8CvRcXO)Ru>Wnh%~i;;{EE3PJK4kURy+lU;_049i?`Wk_Cock&Gc|2^&0O*j`)CO@PK zyO6Hgql{JZ&wP0u_TaQp!4b*O+raP{ve(^!Oz*b2u#`M?@HDiy=N-5Y6&HAsbQqkL z92}8?w7?zSvLRX8**zG5Ww^=}7zu0vV}KdE&x{jJr?d>@4&{WIy-Qa_Me#D8a?e+&bry-jF4lzbgdb|Ce{2BZ;;x|I4)O} z8Rs^djha5~QZ3Q}1ju=~$<>?zuNPLoq$Gkpx)LC$*Tv2lE!7-HzufMZhu}0ue&Bh0 zsCg#}tUaq!Z~(q4BhH=RiYrx`^tCa8zag?VQSGs5>o_j*?XDg{e8O&pQUV1Q_P|Xvq;5hX0Wy@##@#YT!&z-H5ko)q709=d zX9u($;mSq*`k|pn5juf~8el*vvWU!{;+6hF?mwI6y za7k?nJP2Y_=={;cs=HHddJbC2o$3b%dwfax){Cnbx)Pf5EbvUATZRoTCz!mPPBz!^ zPHzk>&#~jBjfK-NIEWdEcl4MuGYY0^38nUBA_WdnHPSv!c$`$-oopn7byU-%eAydV zj_wDo!(UyeEAAd==1}zV02JTKDZYa!ewm>75LjxD^mZ)4CDq)32&dIg+w3tK?qKbJ z_b-tCL|$%*o(fI`gxI-7g1{GOlhd0>jcC#bev2aRb$fiQQEeF>a`4c?Sv*E!5r?V<+_WmoN-h4xb@TH8upJ=$}9n zNJ=!>6}{Bjh&l21vDTl6e)Tucp zTG!w>C426Muw1qt3S+LFFIL-1puD;WW;N+E9H%g*+n}W%JI{SiOP+M5&N!}?4#3DU z^GI_~SIp{bVHUbj%aYzw?8qrm{cv5S zS68jW1cx1ae8i~#szL)JzE{!g+80@SMI`o`SEVefuIM@PKDh`}XF`$`Nn@1?++v)d zR?zK!;h;v#UG78bV>By7lY~cpvvNXWzBq_M$G#q~p*reiXO?L1#W_f^SiN_CI(fHu*TUE~|s^ zF(&g_Y9#@B77}9cEsA1wNLW_piYfKI_!7S1=+jq`Qe7!fI981X{5aL7b4zx%jEYZZ zr91}&N2L-t9SM^Tq%l((FjFfcIuIoeoZ}-(t*b(G#P^)_5g&CNMSY3U?Q_uW4~M%wllL)|6RSMTyZvF&?IOs} zyS*j%WL%9^6dSr8@#taYg(R50aLM94Ql?IVv?Sc05bK<9Io zG`#%YAIU+^%Ui_b4e{8_AIK>-KwLUDI~!n3SV78t>hzvo_pqjD`lMQ5f)B}2o2>MF zcV%!I1&_e_IhaG{b@pe#(Z?$`FG3p^{OC^!*C#-xUY(3{kfX+dmp%$my^oq`G5jv= zg}g%=2j8RSzy0lRJIFuYq)TF)NaqY$^1z^XH5!tcWzcXyZ`t52n(ANsAD z`4KFihk8RCf`r6Sj{zHT)i;CXbcLKwYX`MP8 zBd~`?;K%QP*>qvBF#F#T(-Rtk^H$0!nQK?b*QjgDLxb-EwlZk&1(2fw-Vo0p#WzeY zkOJAQ(@*RJ_nU@s6Y@eCLxV(i1R9p5kCGqB$t2`|-VUG#yBj$>6`WqUj2~2k! z8Azj${2HK~iaN0xXRo~u7+bwdI@k;_Np*EOi6Dn)yn+QPA6R-%RwCY1+JupVoCDE684d*pj_EQzBGrrN8ml= zPBX5E`A1W!c^%%;asj;oA6?liv6_aqQP&ks91JUfWB5Qb_ID6$K+-JZ#swI#P|j^E zrKdt9wy~UEKOl#M7os;m1Kd^j z(lFZq5*tSpQ~P**OFp*CzTilP!4_aQxrg(wEYS;gW+dR@GX>oc0{#lr-wfp%VD?bQ zHfZB<1j0S8`jW_8%$i0an0+ypBr&dD(c^r76Z`b-u$Ch=a&QB+(7m%;v3()7&=R4H z!3Vt52L1wgv3ITRhw1&?6PeF4z$4W#U@S+epNgO?8vCnGR$pRRY)CbY_^Q4{3lFH> z@h-jHfc=+LunXc7n-D?{u3=3;9E5@%Ll9?VK3g)BTk`=qogsmK>#%n_A%0fo61HYu z)rQl^d0M`%t{@60d~p@uL3k?M1+W5k zJq8%u2zei;@~}an8N3y6gyyRt#u#`Hi8Hf^MboC%i%83te^3hr{(2>FZzfYY<_Gdl zOr;feRS%7kyc4KaZI-9k<$#09lccwX5|6v4`XY7u>fv=ybRd~&_Q+9KH0DHE53KnD z+zxgc?hml?&R6Q;O8kTpG_KlGjD|Pa=n#yK{l|p}kU_S-LNg}yr%s`$WlmM^o3D3z z;kZXX59gQJoq!pgwG5ntpYX4}20bYbDJ#N*Hf2K7!5?@wfB26y+#t$?J)8H`F1gT{cM*Enxb&9VB|VLK7j!u&h5Irm?}|mjSt0`L z>jB!2zw6j-5qbk~x0<8RBVPr;UEnv%+1PYawxx zhp;gSTE`-H;VOh{BYj5N)rZjqdkI5|JGyflmqEWxju(&v`)IdeK?ea64KiX+^LY3i z*7`lcAa{UGids65PUSxX${`x=LTa1ui7FKB%$zEZw3ol}SDVhI4<&mHiXQA;R5wwF zI5TgPN0yXtiA;bs3?(3qtvEED7^)4Ic$0oT{c!mnWW|}LQ6m9196;|f<7l)K)xu3o z09~MS{kRNEUiTFEu5o52%M)CZ-S-65=fY0n&p4cDljLb87`;#6I&Ph|%e%x}{u%&r zu*+bTLYjyoi-60{-v3J&)8k!}OP$tMc!tFsg2Y(D z(`4d?p^HA4XAYE*{4KmAY7!~dA+v(;g;Ts^Q0nrY0bVfIjT7SIHP{*+UIMJBJ?H+Y zq+%7Jl4WRUd$;;jClJuj9kEK|{yngR4Sc6~JxLHryxT1JWqbaFdAs%vG~MeX8*^Xn z7XF^f-^nep*EbpObj%y`E&GD`xw^C^@M~ zp81G$Dl=!ff8xxv>D@n4-y{giH=%Pds2yxFPs(@dZbVXu$v8fMeS>Em?3al0dQyGj%BTvKt$2(OGs@XNN^Cya#YR^0t4rb1r$vX1}GZ6dI;e@ z50(tDwSMc*IvpL0=3hWcG24a^vBg&d%FqR75_r5iax9H=vqYXG4L~j9Axz_3xkZ=i zV2|RAA;%-^?0EzLHU~$LwC&~JIMNPmZ6h4zmpIA^yby4A@;dp**pJW>=xqT>dVlCU zJQVEYP3ki?sz2rrh~X_wKkS6--5XWj2atKkX4~OSTOd2|r!Z*4IJIi4RS)$7od7ne zoWRhRLtVz;n}pXAwh1~#U`u_0dQeaSgd$MDO}r#v8N|^LcmW+#%vK>R+fK|*K=&w5 zgH2L*bbz`p%)EB&7DEK*I^nV>bBU$mdbnk?_$rOT+d@2Q8D;_AiSBrt)oodSoUanZ z2&3b<`F_B5Q>ZVc_!7a$$j5%nAU-9~_qC0_@7xYBYR z{7*-f)7W8$4-;9Dgm;7#mgqk6)jjM#E1)3is6*0E8k) zohOK@Xp5d)9m!+MN&kUl?3=Gbb6Q+e^h!+mR3vKn5K0}Y0R_;~6w`nQz?HCNogC?1 zwv~GKjF7-io%UmO z!x7cpfl8aHSq!45Rc5PkV+aKl&r5)y_Ilx_@>f!M5ty4Hu?=UanQ+4HEPyH))Fm0+W5Onvu( z>)_$-X81W#>dm|^6d0V^SCtP{QRJ@FtS3?7`sWmQ>~YUg=cPdP0iDKBq401I39e0H z-z5SGYJM_!hy`Nu)IthQW+y2ae05ohZx);fCVS^uy!lB>g~ukyzdYA~1->?LflrIz zX(&^aHZR3HFG+hny`_l$v1fLQfl_^HA4ulP`GM~i0=3?4(=apv>9v?`EDnea3j{$1 zeuSr?UY*^P6rj6x$r(j9k`nQOSYMby*|67l2?5~scf&y?f2+{Xl03!Y)woRZkdK7& zXZf`JU!Z%hCj%J;A1BSW+C1rlgBQHM*$9NX00m-@KBcCfW{ zhHLCYBixLy+g`y4vG&MAgB(Vs!CZZb`l;eW^-Zb|GcTtL2Rr&Dl}c?^c@w-Is)5f6 zW?plmeU(iK(xI(M6o(CqOvr^2Yq$jEq7C^aUxAS_UrL{wIRz4iU`L{IgkgN)Q!PmK z`lj-iv=Epdpl2!IX&#?%wCY8HQqB9QO7-zz5%~h;RCqxqHA=nH=pC)Tz=0Jl<^|<= z?=h+zu_2g++Y25CZNvI!R1dGuN8Ly+kUTq34^~}ksaFtYcU6`rgD;R9HK~J;gmstL zf>#^gAwJ{0?#Vn#nXU9xuKK#17RbTrX6Qdc1+W`7ph2R7U1qXb$tEvV^T3ZE6Tu-E zc!N}S&eIc<_xsyo9E(k4>nAla05Ahd`Jj zys&LNg23>_Pw&@8?gk_Q@02`gVRK+!4eDI~^AGJAsOBJv5o!AX+gWJ8ElL4aP6 zzgI+vdieTH2e48CXE21tQtfB&k#H2p%5II)ZLESUjn!?)=1eGzl@pz9E$u03?Tgb) zG?;ZQ4Nhs;>@;{-9+qW4ZiCC-VI%J4&jmeaB}<1>T76)$Q`%jR7{^7t>@$e#80w=Y z^rDl`N@X)V^`HY}oXlSg^-gKxG|z#*+jLvMsl=YhK_y^`-&1`7wN**VVbn@;tZr*1 zT#3RD@gApi$xqdo&PgYccUZzE)Q`G9N8l$npvopBTQGs3`+n6ao{qODZP{f0LRFVe z^K@<{a2yk;9cQJ|PUPN)@;*d)A6|>{8W1}yTGX=mDBiO+^Rm$#g_;HFgcqr6;AJOy z=!Ht&ZP`mN3wEEC7BiH55v8^wsULsue&=tKN9 zB4{jCkHC*9@FS!@iL_04N12PyI;EuGgiatYi+&{K8tBHWkJ0;-&O0!#@7(%QHZXf& z%E}o!UHk#)eK>AUm@ ztD}I{(5!S+Do3lT{Z&YW)E%e}MU{@t<_$0V1XYILgLrQ+)QkV9>e*~mh3E7tk#;|j ze-PE~LA7O{pcTQbRX_tuL+GxHXh|t=?@K)SgDpt4AkB>5CUp6kDahF}8^vWkg?Oa> z$|;q#;kg@VuE%>PQg+~p_tIWGLHK1D7Aw1{Iux-|qnC5|J5ITE%=@1%i)4CkhSjz4_o94Hba1^u}CV3Je1h_b#p?37O9c@AZ-0cC%(Q{4m4xyV+1 zQP9jr6vf9&RuC0Qo+mNz`k(`P-Z|e_>VgCT?COK4k4hgl_5g>zzjR*GJ5lhWY-p}N zN`|(|Ma!zC7{u{Z>R3BZFL+1b1~4Q9uIFrJIsI0Eb*%m*6)!z%^91m?$}EnA$Nn2o zNl$!6Ll5nL;@@})JdQBVBGA+ITRLZPdaXC0#4A`7`ePRgJcg1#qLMpMf;=Cn4l%9; z(Vn%9%!8U@j|+%J>OZuK!5ZdBXN{-X)w{adLi@dwaD z7-6eB0n4xi;yrjSNHd<5mhRa)206I5XP$xj-191e+D;R}gpgO#0@#nBJJA83w7^9$ z5>#Ihowp6|;1>G~z7plc^bm-aKF?tWM~HLA_=ptfPr zL}*(_hpM@q5^f!ful_N%cETz+eAcZU2fpGN2N7l>3pQg+gdN!gi< zhf(>;M;$HGm3Bf?H@>!IaX2DFstV5`O=`Pa$9v7toj8n3f(q1PaBd zKeq=ELZ~Q(WUyc_p5SXGU5Gn@xMP?cF2l%=T%MeN-JMq_`^y(@LmlCk} z+m8?uNHD8WdAsWc*=yq0pbeD9OQ904Qc@%2)&7#U;Ohq^ z5s@-JeEGy&kG9j4?C|q^ff_1-U;~ZHBPemJ7~|0$cn0oAGzHQJK&cNb-~lU-&IVPS z>>pJF3Pg{6(*@BRWmm0At!@#d^dQNaol^z^lk<_?9~p$CRs9vwE^6=_(s%apGY`y_ zgMqv7<9js?CF??ury4yg5!f5|wcFa((sl-3+(p1GU$=erSqtNAkbUOw^4y1HQL?d( zeRig6XSQ44_Fdf9vc9b&@Gp!aX2TGPhqccwU*irPS>N?<6n0emM_>~o12e%p+gsaP z4^PpZu<1kv`i8czk>6@u!1SYnfrn`P)1XSTNM3ofrH6j9S9oiS|t$_sO#e;w={XPg$ZYd1u6-5~X zpP-H3Z-#+@`$srKLtA~@(Zf^50qD=#zTdK^5qY`4wz{VYqirp%NBhFsj<$7D5tm!r ze*byLi|)~F^+ytJL17ez0ud%B@KY`gqypAyJ!CKQ*-C zc9h`w{}0u)wTOD!z6e#1fY*7&UmW=E!0x);sJN~p0*V7C4xEVF^+Q;gA4tP;vbBMc z=TfbUI+f-yda#DW+i($4@bK&O+vWKMezS4w1s@f$A+m}(5-k&9wzUkD7%q5dOvU25 z3H}oGnSa1SQ~%$y3$nWxbyjVovUQc=vg2Anfiw<7T}tcaT^-nSCx$22c`%(bPg9RUR*~s7nc&jHGhY_Xg<@RtizhUJ+H@Ey}?wemkn*dG_~~83N6Wr z6ptLm$n9wB1TXNneb?UB0O`^Q=@P?S*;DDn%p23TA}yu?J5V?^9Fk&Nm9$5{M?z+eAA%bf{~(I-E<^j9|E);?a^<1FX5}SJ(Nqwlk|KVpd^$a z+NCemhQN?UGqz)ABxD?cQXt)SR>BE++fgu|&S2);B=_x^Zi{<;8*4pCzR#l{k=olp zsnHV;=Ia6spi($h3(JRO$<5NQD=&!r13kKk#<#W* zC)V9dki%Vyv;uekLO1cOWo4}g)n;C`dw73qXor3@CXbP!(jG~o5EC`F0PvvjcHFdr z+dim<_EFsEnQN{l!pab??DH;Xf@8pJ#uA*q9{Mfe$4za9$JL(cxyYZi^-qA_VcjYH z`g!nY9Ld8k?Bb+f8~EpW#JcqNOxh{c(E1yPhMMqDShQ605AjVVsr*}^Z@ExmLnP05 z%nwbtSOGyzc}L!;fL~DD%k8$b<7sYR($Nk}q1N^#tUbowzN9mFkhX_C0o2Go?CHgp zARHrKvRVF$e+75uKPE${B7&!51iuoN`7dLrtLcBuG!}Mh^H**sIfR!Z5t+hW%lj4mi|zt$bbUSSj%VLO@86`m4x#ONo$ceemyajPa&=GP?>YRv zg1^7v?``~9@z;RAQTR*1-+ug^#os0T8PRmkg^C^9F)P77V!8#Ia`w`ED%nJKL+aeb zEnKtncFXu?KH8ktX7>X2`y#O=QZa>YT^vofE{;c*Vx9z7+IHUsdfsKx9&!}3B2gS} zt^XOZv*?UUILe18-0CNVFxZ$Y-euD6VChFuE%+L&iFa|XM+|M4qrI)Ty_K1HA@EwE zjf0axcYq3j6wp_Ih3N~Z$`oN3)31Mq{m?=>-B-SDQ7(4IO>E1xQMx4|`G+~B1E>UolV~Q#QF^cR z+FN_8S8eV6`mpr@(of9;11Qyi)ItFbiuQ~{F)HREnE8Hd@0pN@+V=l_eOj8K0Ds3LV<^Kz5*7v9b74rv48)eBP0>r(O; zIDH9v*)Cu3$4o{yoE!~MNJ`+_&H(qyH>E6EP*&w$;$Gb9uthpV5O=dkI+WhWT;W1r z5L%=&!xscdkZ2_7a;HsI12h#4MXYM{%S$rs^PQ!&DjO7ucA;TfDs)@{h&O6{3LDDo+KZU zE|;;`Q-Ol_W%%TL2?xkgUy*?uEQB2-(F){5Y}R^0e&uIzA?=zhahPV6FPNO?I@HM? zdzx98)nbk7fwK@7m9|48Yrx~A#;-4iLo~U_K7BsJK>Mlpxzg`&#>>a%oJi^^z~G%L zzpMxqjg;-lO!u1n1>$2EVK|Qi9fRLIC)YY=7rs%~qfPnfRD}5-y@DSktr9#LRZH9a z7c)NCpGsNLAp`foy)tmdNOdCL@P>H+p$0_N?z&5IpUUYC&Ug6x9IMtKOLBymu3)jlUC_0*(`YrfvN5H-y z`bC-pD3!v49unFuz3hX?_Z)B|o^hZAk#gsXE&+(&6ZWlR9gJ-l)ChhDiHL7KZ518$O$59 zEtmQ$>c|rO*?PWUrM4~bjO_i9eoJGs{uGenY!_%)M;x&5YTYs{1nQko1!7ofk!PgT z$TPCaTAPOE@P1!pPaoJyk08@bCHyN$Sv&Qig_;qX8d}24&y{e#lyIGt&=EM7GWPi< z=4J7$RglX$Ga5;H*elIsJG5wR-5hY&;&L&NKNZOMV1SUlbSW z*XDJr4h61UV09b-M-E7yL*B1prrzp!maAy(SNhy?0gvvs^-B_}qJ7&US6WBHloz;C zExA&g9396i=mpiUC%%k*n2^9s{G8ns%WKq6C1I0F(wG#wi5Goo`P-js@d9w@(e(^6 z#v`XR^3;1f8FKkjN6e%3@+03Ky_X+wDr<%7H{jHz0hwF-5qauBM+5}`F++6#0ZM;= z^2RcpGqro--8n7Zi}-W$w=}24S7L3c1>nei9l(mgXpz1C9;mXj=$KUXi2Q_?M%GqW zSPO_o^h-%io3?O;+3vRt?>DiN;+;_99c6W4k~}JPw(q}?zId84q>+!vPxyS^q}`9m zPdJ4)Y5ODcgXdD(immFS+t>JWo&__s#RSoFElkRv@HNH1m~enW(u~k2CMEg0YW>ZU?evXlBF+9K3Wf zN<5whL+~rD-%D|A)}sT-t%irdYAi+k_-^2Zu1lG8bPg{=t!K=TdNM4}K*rFs(qF7+ z{D<^c8Q(Jerd0RU2DnZ#vHh90u&2V>@F>hAusL8HgEF0M7KIe+VqoFVr9$LGAu{uqLQG&pBy;xc0&0ACT$gKE79jEF}#baXnR2-%tidGly^3y@hez7{&^{uLL0>R zdy|Vzm+Ay&IM_KMTE*9B=c3yG4U!>$ecV*#S2UuCuc8_ked389PrSHx4}S+6 z?kGpBRxt9;F&m_vrfO48s~v;NkjuLdQKcvTx?qDexD0cj0^by6jVD0?RoaZtR%vqh zVg#6;k%(CGcR9btlD`2&qFvk_&9GpBCn2DEEe*mHg{}ot-2$YjYj)Lg0r3_JKwV#f zae)Q_``?q%F^UeU8k9W{3dwN|jW-p}nn6Kq`-d71`3iN6d*{iZJumR{ep?q9RA0a( zv5RrCz361>R}}wu2FL%rjB1>5Zx3}v8l;xFrW)PQ-Fya}1*hYEc{rfMC0#*KMpX~L z@jxpYPJt~7vDBDzTc*LGQD7kRw-;DX{h49X1&i*74`IlBg%{q7#PC8uLCy*eO*Mb6 zZjq@<*T64WvlO&8SstXeH6(frtN5uY&B+q|de%FuI<(sXs1{!zGA`P;#eAURs ziqP;02qjYPzZO(YSX^)l19Akd$?pvi@e}U~VGgwuZN3a<7!Am;+n}k&-Rp@vGwuD4 z3u3aZR`6-@q++6Di$pEO@WpEIDIe@X@2sZadBG{4%dfBxEwVg+TRf;#6DZ)9@TO1u z1suC{w+B z#q7vbfT6Rfp65!>@lo^X6A;!_EqUF+;6vyHRaC54#Ljs^Fep{zwBSNyFGfFF$||Ck zRg?u^O)*qok+UVC^w>s^l>jjB9vj|Mw@8n#L#ss!wzotH(hx#Ch!~-(6f7izJt)~U z8A9fia6!w)--XYQPAwIK9Ti7D0ce7(WX;ay~TM7mJv4-k+9Qhu+i&D0Y%;JyR;p#C%L7%tM(@&Mrm) z9Qhop?N`u2br@15e*@mMmbpX&fs@gdR#Okf???$73#^s@BWEwsI{I@UY461xWuv-X zM;vODUTAas68)WT;j0EkKiSEaIwSKO>xP?Xsk;gTap7SLtUTFGvn#<8 z0x32f7bTQ4TOjV6^7gR-5-V*f=3fKB44@kfuID$*zq`!5`_$XHYqa5)3+Mp|v#9 zA8Eq8GisneyImUo%u4mUTz-S1>LqqJPZ}`&pKUI)es8CtH2vypgAFRlGxBRbBM`!E z>_?sfg2B5jV9D5B{*j-qq9~Wi$mPcl{ynj>hM2B4p~pe8wwb4VmXf)c}|@jFE*+ zsrhMLLVq)2?t(RD*=bgH0wGaKVonqE_g-t<|6`_)_s9O;OYkPuJJ)-!zjq!UCwkEm zkz+*v&d^#~)l?jZ->nJ3`Dhkj;k&@>izxjz|KoyaHvi*-X|_-)DRIALsAQM>lFYu) zG>KZywpN}$@U9bYTNg`sol$?GJga5&{>P=`w#%t!9RVCS zXJEIi+?(9qAV7{xzskogu>_fR`cTTyVbkc-}ikicl6!q9S?JeNR4b|*1ti; zH3GRc28AK1c2!ui81bL=BvN&#tpgtz8_#uw(nM>;WtFpJuj%ksne+cEeyv{ z3mY8I^6LZ7OLs!QU?s}y_Me(8vhmjtGq*7#oZvr|x(5N(qGIb?s_nX! zZrufZ6{?iNzwig8;UF~B?@{J1o5N^NJZ8&0*mQiQTT66ZY?*j(9FvI32mw~D^*rYC zl$P)ijl?pocOKQtJVY0Ci;As3ppI@Hkh*k3tjnavuxU0p$T!Ffb;Dxbr?nXkogng{ zcHnP|h*)vom*Nx|H4ikYEV|f6$B2K>zF+l$7^2H#)ECe`)%(Z$`y=SOs~!)k(!1-x2-Xt6q5-^sBn*4C{=59#_hsZMD`1^r6@3OxBf)7yXQ2FzETDl!_x-o;t1G3vji zXn=sb-H~|1Kj>#muk=$jrhHIiE>Vpcz~{-neBH-?Jg_Uz6c|r|8u4<^YNpRQzCkc2 zoIcQhGQxP(bt-sI`RxA*yx&ynNvex0VI#%rD^~RlLbdkOF?i1pDC`eBznm5_o4q4+ zrgF;-Gm7OVHXfOj7oChT2 zrUqFJHO&`NDR0H0)zATE;mFx%6z}xBZBm4+zIg*AO_rfJ6K@5QUutApszo&z>i`_! zn~5*C9y~4`h^AHaLaW%>yyt2Pr5t_fIc?M*fw1VD9C*%@Xe9&Eq2x}^SpLZOae^6H z?K{VRYS?;kG}X9Ij%fZ&*eZoc9_nl$_Mw~ZIBNjMB}WIpHZ zYS?Sk&n8h6s7Axr)r)ix^iSqR2CVUCnM7FEeqVDGCYlY>Hr>#|fO-k2&4Tqk1P9%X z4deM7R3v1lQm95C(8j4uXi8+ImxBobXww*ZSdYELv{K6{V^gCwuxkQx(1|XxKb9j^ zi{Dy_<~S{q+xB{qYZ3o%pYcsaoG()qhoG0SB`APSy$w~?L)^b>pm!t4Wd_su)!V<2 zn5jN%WQ*%&F7-K!+ZBv3s2waiEG($+?N|HOx|vIih5#MIRKA?r@D0TJPtx>Hr!5xicAcdE6C%&e*Fyw}7Ke?&x4bYEh>+BSG@bL)LU zNSQKF)qbwF6jfW4RE1X1#cKz*Mkt0+S#C{2qDtmkH4XRVF4mMSH0kWt|8MjPqpcaB zfL^RS2_*a$tQr;QjjN3{ab2j2O7%vsF$0vBNsqmAJ+DJwm-CmJ2Dpz?(^k!fA(ZTX zOEpgAEa<{`5<8Cpy!5ao&?JmGZPYL3weEQ}2{lG|G2())flvt_+Sz~<%2UxdQSDRX z^!bsU8XZ}5R-3(!C*#7C^?V(=kNN{g+yM}1$Z^vo1wNm}FEn61;FHbp44yKKH~RZl zdcUXEiv$jGrlM(z7uzQT9l6by&LK&kojPTCwQrwr5jQAyv#&k{54?Pb5OU*-+_j{ZcoUF!d@6j0{fW)`)LX?0o(!% zg8eU|uuoI4e^5c|3JP!Ow1@L=YLMXYaIVUEF`_qez+2h!xI{b3t$Vv-z2IV8rHxKC z4L5&thrMk8P?Bc=P|LmuP)jN9^MLx6Odkb+F9*~uFwh18N;M=3BCk_mB31i8nA`K# z1(6AAM_|+oga&3&7CQ_*i}b9~eTibScZ{CCQM_UZl6gf}S= z7dlKdhMPkBc}{EbC30}}#T9>1hi}0KgzO}filZC2f;L}`g7uhcy<%qSY}QqzbaNIe zMgl4Ob0o}{TsL!~Zx)W2Z}k;4CCQw-2}p)7)n{+9h%ZO(;$s3OAuXdT!pTEO%q0Ek z`AKvo4&y1$TtmRB`4bsTb zcc{y&9v;yv#{Z93Zks3p7L{DNB{u@&f9}d{<3PPup)TO7uiPq3if*&gNXwPmtTVYM zAci3~j91V@;WZ)2C3msdcAVA(q3U&ukvGFK2680t4BuXEj-Af65q7tbT98|<%E;@? z!!%em$YKObyB3IVSBdi(nrm2Q{rWW^i{f=HmO#f!9^8aQe7&Ty{=g@Fbta6u^SNAq zPkDoXFEM&!Qnx(U@foJP@D?Fb51xz_n`KlAen8vhM)g3zcQ|_Mn_=Zr8UVGbRjKOs z_j#y@-2VQIXYBU(M?zWj?eDR(Z+|l!a{Ifo(pr0WY)p!d4P41(aQ&MhEI{QYpwhaL zvSc^jW&SmsZoLDSA$lG97(v=%zA0^7ayWMJ%K_hJz8=nzi{B;KWjRo3J;OeX0a~py zSU*2g#-+BduEoa+o{Kzl7(|ykKeX0^LOJ zsAa$gkH;9&yTh-4DX@FZMOcJT;KrMZfP2r4OoC?RPtDlUOj)T`FEdh)T=?qf0Hj*) zs7L{Pcv?eb-NhQ>t1nd0#`BT7WsSm;i*~1g+IODy>w#?6eYH}P^)MB-&90j!9(m&N z$m7H#&$-jr`Wij)i}ACQ*FD~sk~~&?_k?Q@qzH$x8*roe*s(Ibi~KmUnd!H#^o}g? zULr3U%B#&h2&-7<@qG>7Jdj*lI0w0{^XG1QC-NdB>Ox*p=i&vXkS`fV=!4-n-KxCisqx%z%6VLF6|B2{%ur!vNc`HltV58Toi>r32wf_*BD$ds zOXI#Mu)!5W29CP$V?rZH0w=n{y5=tojEu8lOf$ZioxNE+z1)c1l=MP9jOXBqri|A) z>ecHo`@?y9rulmlVdP$~>XxFdOcmq|fmfc`WE@8vp8HZLLA+GKOJ@3F>pUgk3@2L` z4P+kh47Na|3Qxl$$7wGn_e8d9L@#9stsG{3_zJ|;79NW%8lqAuWmfn(PJbA+?6Ur_ zQyn#;?(aO4R&)p|ZMs3|DXyrN925CGCw>SNMvb#pQ4|#F=RBgf>6`9U6e?6054O*9 zMEt`d<|3EbI&tGn5iY}Kgm<^u`hu<5W}DECmHBq8%!3He>@3;=tIQTyva7qb&SR^R zi|C>~G%r|$9B9#g{thrNNEks!^Y*iWHh4XlVXLN*c@T-P`qt}o0bA0oa3cu%PpG}vEhEg~QUvgla2m~r=YR#-cTa<}nP z+rc{!@g`MTzf+Og%-2tDXF_~IYa6Sbh-wysuwS#*?T$?`>NBSZE+Zc4+aGzu`Z}Mu z#B7x#W05nE8(2p!tMvPthKg_?=3FuRRLUlT9fQxsoXu!Y;?9VS+9hg|YJuG2n1898 zX7f5`W5Sr!2Xo$N!t(m%VtL&@&-XFAkmN?g3hyH0Md6naw0ni=tF{@Ho`Uaix|%xL zdkXnPvwt3Rp^bFd&Cx0W;cE$or%!A37EZs&I~wcl*h!vj_FF$ARWwPC@}K3+Btfc5 z;C)M6M)*F!Kb3d5`sjNhT1!rUyWOVtmHje#7pn^-S!|VsNRHAX?4Tl9=~<~U4TsgB zZY!R>y_$%=QPxGOY9%G(SO-zO7FU@0Yn`m+zLW*l6y#Dx{xy7MdDD5uv#)p0nRcGT zJ9bvndE7;eYI$33yOKNd7o~7-^TS9uxuH93m$!KH!&&u*e6K7XVp}@q7&9{Q0E34t zSFg~A?;=siO(rsErJWE6Uj{eVia?xWH9%1G@T^!X2okRLk6^KYG^rHcH}gx!c(F2? zHgO{)S<{D?u^k!ljTu<6Azu~Zg2m#T20X^dq#TgSN+%A?uv0A7Ysb(M#Bnq5B&fVm zOytvf<_NQeX%r&^1Cyv9U#-VC30})GEd*Z^OP6)@qbRXVkS$IQR5DXFNbe$OoEn~< zTVrJN&Q>9tnPfXVyYMIn@ zKEPTyftZ}SsKj-6o08ZCM`D)wCa%$fn1V3T-Hj9nsoUWs+a{G zNGK?EtvbpN&0BR2m08QE1Hrqws%_ykruFdDJ1{lhpSu<>hcHcM&UPQyVi97)znPj`Xh^ zgK6I~>zE`_eoa9KOLj$b!%u)s5IvOvm!=igzo~NCVygw$2I9%|VcHb+WIC#mCJj{| z6eFVu+hflMt&NY34UQ}ej_+V|J`*!PQzj_J=Bj7q-%+DZ&nAp6IeO&Etj`A2$6bBT z*)o;Zdnlf_mThQU+9tG4wwhw^$*Bo~`D)T9ine8>|sGp<*O$E3;pdg+84 zBQ>H=UvZqcx1@IBu?JI)0iXqkCe{I>7AvWTH`gg3BgTe4jz@d^b!5%`EdEC9HPtj1i>!m%q=N2&l-HJz&x1^5_fXpz{c(I5kSe{9w=}{ILLo0=B7wAe-sEHl~w3rBgK~4HU~x?^5CHi8h6TVqhnGvW$`% zI5$b$UHmmuvy3N$T!rZ)H+^)20#zqz%}baga=%L86Pi^m#(4HhfbLXdmZ185{`&d* zK)N!teUneqK7X&n@Qco$HmI<0Ms!k4oT^%sohq%Ry3?`NQWhZ19;;it^7I@=_ZUsU z1kuV5KtRkX@WzOU>7h}n533Hk!H3rNILi5q4-EavX7Y)pB5cs;6kvc@ZdiGk57kF$ z?I99is7RnJy>Y+8M*)2`__Z{w093^kb4zJ&%aC-O87ZkiqOcoP^=JMtDkTGZ)(D-( zw8jiE$YI?bG=5W!6N*sE9*MiF~bH*>#wsjESh;4#q9u z{Fk_v_l+JbuH5=5-HlNGsPsAo%TWo9>bX$h%bc!i{et{hB8nzn!ShQJRWcK*+P8N> z1438R$H59t|CqkI(u<{4V>8uL)7K2bVDrC7Uta}cf)4+PzSgBkjZykqr%Cx>`YOgc zK2KknwWDV2N>fdpyp1O)_Eq$?StU{QwNJrS(^pDnidG3Bgla1zV2ZAOA`s;7i|OiB zhF`o7NZ+V7x@!DIOsaInF?chcRP9>b!PF6$jhZO1Buq6zcQwr={5Qf-tS_RhLkOw> zh@=pa?6GyaBLjq$Mh+pUQCM^muL!EZC`M2P2&^b${u~AJs}$O;X?&-X@INA~QF)Y8V#eJKrH?Gc-Aw0la2syhvWlbznF!r2qM)YGkvs|YP}vrY(fhR{ucQj~5g zk|~t3N=(aO@?`Mrn%u7_W4z&?f7alj z-B>x3L2`+aG@)$GyhG)x1wH=G6Y7?ou6wrf2+gV%utVBco{~074tc&ygcx00D#=rA zYF=eejshVe1_Y{klioA?-93sXl3PF`L>bd=*>CWU@f>$A;exWj^=Rli9DxF?&Hs?)qvyxK{yp{bT3{oJyX z(0_*L_9!B`K`hR6WI}yg|1XK|{a;RWXZ$ln_YW9yWe~rV=#EN_(zJ8|8z~bS75HRv zSpNR)F+n3QsOgZyb%Ekfysit{DDB{MHzkrmrJz^{o=B zldpGmgYcjQHYeTuWwXC@^Orq2m#>7`MpjME929m93fl6NaZqO)u3QZ3fCbq)%53<@_73O_k0{P3Xg=0V{)!v9R`fFHD@ z5)yf7`U+aPdOk4;%LwtDT&SOH(5u+5u6}(I_5kVZh66!IA>SvxwCz5XkObB2Jlsb} zIJSyROj>44+P?64+r)ecX&&Ua#F$#V?5J7Vz3n~bfEaI_>b8$z%LGdS3F>QHH${uH)ZyP4CWL%hfK zNX*YoJH*m(`A-i%U(MH_`e$sz`aHJotjew2`zw>gc1v!&7w^Wx&yg*6kc#J`yTV7# z${ovmf%Ozhs5*08-Q!}`_%o|qm!V4-oH@!l`3*x9{+ALjEywygr76T%U z(dG90XB_`C4ff4HvPC-5r1W)%H+#>Y9`P=o-tV1BIFB2JK*K8O{U6ATSda9zM$FY@ z#LU$gJ5>TXe}{tn9Q=)f2X=EBzDcQ`TUO;|x`3vuMgWU4Ys7yE zuv4yIdZef+G?M&h#xgn_7t%0>BQ^lhK|J)d+>yYxVgQ7=?>JdrV-}65F%KvJ0SMoP zHJe_EVE1T=c#_6%knz}od^V>uW#wobiQ%hHg(Qm12iNZY-s)wN~}G|tuOe|x5^ z7iD}5CS(HNmbVk^ERSx=sBVKIrZ(y_*usoj=OlA8+2oM-Hi;}Yl_budlB)}s{R9?2 zC0=crIPPS#TFM4}R8-G9**yF!safYWlH`)14Vh%fP=b+`o^WkQPO^+1cXg_a7+{v&)9OT^i! z4`-@XaY~(jra!5lW1H1eY&nXF-|hDoD(lairw_LNoUQge!c~zm%;ubGE3574BR$lnM!kr zlhbWeR;uuR!mE3}K?bp8NyZ9xXX%ouWmD@0h}EV~*P|7?_B@J`6FVPh!r(2R|M;k% z9$cS=>?B5ZYc{=Xv)8~Xz_6Ivv9~ve=G``>OjY&PM+q=Bz;|J)X(P!WZ)%N3G0#?? zyn>P)vXJxiV%diE`S9*H*)w0Y0cQ{2 zwQ==jvYC+WeBJ7EXktR3v1GP{EG+UBP^%ix{*xW z=|!Tu2e9p*;Wb3@j44utg?ziLP=n-jsr`UarD%#u?7QVh6#Ujx1qmS6ElV znT%EkT#C&J*Ki!XP+Nk^S3%W>Q`l96uvj~X1<40_;sF@^);12XD zn+_aV@}%z8IMuCex)qK;azMJ)D3?5XSLOCyLf0-nLKV@jQNic>*2M*1v~Sx84)kp? zI}tV z>vH|+R7jjybhn`k^`CT_oVb1wZm`E`86-hb-u;}7oA?PW-i0%4Y;e3+&cK+V&3isd z!r$Y{=HQ9L=V9C~VJha|FnxFDHQ|zhSb3kMKZP$J9DB!w(bx-z#A4A(Oy-57&x4&u z&hx+5AKE7^Yl}DPck3qNmHkCRSPkfIb9v*6DdR}o!4qP-4?977v{5^; zumtWioC!7*L+~YfWYNikCuY2g^lEKVUgXW)#o@{~59aOD=I-KVn?;{KXGfOS--&}b zR}EVjvKO?}em!TGS#+|y=p>T`jk~zwy;gToZybgty-rz0>Im(u}k#dI`#uD+`5+{?4W$o$>?hI@l9lN ze55Eva0lWVofgaq94E(FTfsaK2e&U1|EE8KppjferfIuo)>uj=r`xUX8g+R@9Be*P zly{ z#^_5(j9Bf)ci2^kJOb6q7Xe&zpu|ZSb6s4scel8_$-KSDFg6dXa&))EEV~+K90V1jRbKTjOwAS?a{MysNv_Ar35^ z{*n}DcdIiNc67J8V&VAi)=YeL7deMj(X`k!o1&-LMS7a0FwMR$#W?Ml8lk;${?2~? zu6Q&}k@!)spCz}2yCg~GX(@#fW3pyqvYS~d*)LMx-a#}^r~jLOmm{1UeF^(_Iq}OP zm$YduM-%d#n~rv3Tl+E@k6~4r{@P5NcajvaER#FmjDs?f4pt!x!q;W|noGv`mo4Ly z^Pv4;RatnNlshX^5zP3&tW2>8DS3_zSa8WLML9tw9dpdNK1a&ONH2#^EwOty<^vZ7)~-%UYbJ$q7(?OEP@zgsg|4f z0H)(&R&7O^5VB(yWk%R5(P$+svA+qhh>EihC+BX~TsP)zmxeEh z<+46uA&|*BgKtNQGD$1qgLH;t8}tU#sg+_yaM+Q9D0lyk##hvsz7fXO_Fu_<%eNjX zj2*9@7NnzaoC@{xym|0Np`x0q%63Bp7P^Z@xP0U6h=-&!?7KbDzdMeQecybE_gYe9 zV3G5=41 zz_ICXT!q^(-wp5@u6_4)pXu$#z=P8n+@a7l{$oGjx9XG&79oPj<-c=JQ?FT$j z0Bl0ykewZP>PrHC>1>;HRKz!ocQF);kWuM~?Y#h^X2Q!g1uy#?9(y>XFV=V)*;AAv z*YdyxB?ylr{HmC%!=hTDNH1LWE!idlahY6NTQZ{WDgu65718L=MIH?II^m z0L5USHW%nk-ZXVAw9RYBK@fly!)yMDDu@3D=Zlg)Dvb~x?(ck&6;Gn9R4P*aKT>=8 z(A&m~7+=#vyx^w?YsEh+mG|t$&2G(v($h7`*cfaHuu$BRGB_!)gTTglhaZ|;MiUP$ zasE{3tqT^1jsb!{%$`lc!TmC49kasCeWx@J^%2$wVh$+ z{E%(Q0i6TGpE)qR^e2f)tz)Omb8M%6PTZ`YQyx~&g`6QOI3-nel3|B!ViyQP>W7zI zta5Ezd5Jt1c5#-1`{?EhyIL*|I2(zxl5BESwR4}!#@!!^+tl|N*^PUCOJg>L1dHrk z*Ie=;jp}YWI^5p!oQp52tzxNvjvVDub9GIee6rIERajl5b5|^@jXR7= zI5f^5!Tn>+Beo2A0_HC+|Yh|ybtS!{vx0*K1P`+*u5Ej001qW`3=cE;ww(^(GETa6CS?u z3;3f>T?S0aui%e9sSE$V@pRWF&#B|GvQ6Gwy!6VGkfAno$H|g?Qyg)WRk>vbT15HVH3Jj z>-cQn+j2DCF8p&=(ro6i2-uC9@*tmD-gX2N3RSNbPJUPwm3=^1<<%q;zTi@pCGk_2 zy-rx(+%n1)aJ96rB$pQuFnifD0-co3-J(z)#fzNEc$Q@96jjU0c!SlMU9wWgs3f#8 zyQDChM83jM7f{Pbccbtpcp~E_0)}#_??P5VJpH7LJcB%C=|fzlnlx2=_zqKX)Ha6V z{cdE~lrW)Oyb8}>#Qu;Oson`T5d_|YSKo~@2as$-ijPZl<%Mu;vjXy{vDfPkFDP~xrUuw{Sr>K7R{krkkB|LC*KGJmg2s``gn@6KK=}P0AFf-{61C4msuZAfk9G_=`^?n1Ez=V&WJed zkdsYXUQvo-edftQ==7OwdW~$sl_;CpQZu2orl8d~nda)F!jzNJ&k}E1i8rMr%KL=x z8Sinhci$#QJ6NAv@!Do%RBPDPU#Pb19A!A>;k?7>Ie5Wt|Iy9CO)~0B+9-H{<8()g z{@59u#nC{*n~pl!(*!q(S)S;L1B=F@H)q{PL;fg(WOkKDrg&wT?j)+ltQ`TIqZZ;{ z6AST}hu6-Kv1@fixqCP=j%wxLb0cS5HEP$0&G>O*Gk!LoD(LIEeD{p8TK=xaE%a}+ zmM`ENU*PO;k|RF{P)sZzS>m+hv@G9uWQ09ZxH2h!pamvnG=nM*Zs3WK8kO)INktJI zcL`I?<#r!&9liyITJMoXgCH`>*t%28I}TIFEC#O}L1|SPV;{8{ul8J7XPC*1GKUZ# zg)Lp|JmZ_Czf5clFW>~r85lNeSDDog+efS(X9`1e*o!iB=n-5IEVA_X*1O@ax#1W3 z5;%MN5?+)6gTpDW@xH`OugRd8KF-m{%|(aJ(#{+d6{ZEQc8IcRV3iB*abR|u*}B;B z{+r^n>I1#Ye>_d;o#0A1;!PE;v6&Qv>*zp|I8tsngyPmOemtP~)HoM-VUt+o=U~PK zOOIRCPg7peak1a`N7|JCq!1ThwldB44ngBt$Ay$uH~m6%v4!b57h7}%KG&-Elxqf5 zJ!1+(l?`WAsR!ElQO0~c8BjeIoh^`FPVqdM3b|AxA$woqT^-8vn^%T z;c-Bwp&2NM z^yG5uU0@{YBo|jy(74aF4IRS=2C|0j!IhCZ{I&~=?4+poV$n_OC@MT~0Y#TUSe!L; z`-W-aabX8wO|#}xh}IgpyojU1f5#d^PdHD<2Xf&#q77r6!hMO^mcJxpEouiD!pPXT zW|%cnCusVnuwT7))3bfZYl`K-No+fp%xa7}SP2V&SWu;k9tjfrt_g)2IgB|4184d{OypEP6+ zC_tdTfL9`4mjEPCdz^kI0}C;Wn+e7%dv8-|Q&i3Fd>SR#=HQ#cw*A(L-_xB^(0o7o zdG%t`B06eUh+)JBdEX8d!an}F#ytypYf5T#TvlTyfpgG5qkau>a|=FR^=9a1LjHB$ zINx^}lZ#MLeZ3|>*Nf+Y_^W-N!01Wg72910a^Bi|>MH6BF6z3jqI^L)8jpfT%H&~} zIII6V_4#YNY(|5iyIEw_tkX0v7ml z1*cYR3VW@KIZ&UgV|^c>4w@@Xhw=>p1UZJ& zYhMC((a(BuXei88Y(^AJ_p=T zp=iTeK2I$${CG-{-AgGZIuS-vj!cASK&H9^oA(`75#V;%gj9Op5Sq8Q!c$#tJ+=$0 zAD5P+U% zZfoHV{bn>+6bwWm$VwgTGhS(+*`+SRDy>xvslPb^Y!(LZVXlKs?r2lA;<8gZFIY|1 z(-qbW3kDkTD~YtGklEhuW+!F!-Ot!8g6-Fvv)BFx>#YqxY1``Akpvc!X#qy5E-1UlP=5>So6&2QXGqp;aD9wvrcDi*HMMuYaA|L5)xI|Rf zyzinEc$zIhieMwvT1&Vv!o-;vtT?F56`(K;fBPHkA7}t0zrRUZdD_~;6Iy&mUC)0! zMcA>aHhzP;3uK5UuvN-%{0vE{E1C6r*i|@X2N(W0?Ssu@iXcO!^Jc^l_xO>WLRX zpS12EB=ctcRNPBAt&5pGXe@KZ$HIita>IXvG}a;#`B77Ecz`$X)tdif+1&`92}NI# zd(Tx|N7HZXpGLP25*0f)t!&I<9Y zCFo)~UQ*Zsa>xQOo@o538ujV&-8u!7puP4kcT?i@YTpD;b)~hG3~oG>c&}LTSp8Kt zReq)Q9Wa@ugvU+4)O*3;53}?KY5ggMjlZN!nS(#)Y46%5&6DB`|9|tjysXkP&g78G zU#kf8)x+!0ls2<3MLwgVHeShzP-=a-c=(C zNVnFxDye|9YvQiO`da(a#jP$a2a;0Gy+=i*zt?TtpAT9Z^?iUIE1|~y{eao1-@z|a z+oRwfeuPK!#Dz^dV-9cUg(|GG&xuBTP2gs~55vGaA3ko4@y5Nre zKG@|ARZx+nD{-+QY0Q^&eaXiCxhh=-4jN@(pKcUur=jmsn@XD@X;W3&MB{$P=hAl2 zme3SQTbkyX_#HE@CcwQ;hCePo;tWlUmZ96Vo7z>onhAJPe&X)w!%a)Pxpz<-uM7P_ zz+jac^{-R1ClQ|`M*SlM@wHaQ-{cabVKey$-*J251r1*i@F=^H4Zoq0A&+zxLkF|P zlw{*Iyos6RIXs!6V}m6T96$|xo*ex=Q8Mi^XOEFQ8NAH6h4vWtiNxPah@un{gkJaZ~eh8BT}2HFOC%{k^vs z_cxGClNi7C9n9!U$ySn4k9RO}(05Yc=Uq|a%~R-7My#q(_40DwqP-OBQ`A$ax5^%S zXt$6=`n4_v^Y_E}N@$#MzwBBE_iSZ~RSJM0^@=3Rtt{i?EmV5noY`0ll>W7KzMzq| z?fD92_`gM~TM3v+o=i7cux#c}ijRz_s0?KZRBAI=6qlq`LZ4k~={TsWQN}@=$d`X5 z`)bK=Ka<}cAHf7%=$LN0!u53v!1ImQ(5(R+KTfVtZ4?kC;P|@9{=USG*@M$OEot1* zG==O>)(!LbCL8yq&@^iX)FiqbjhiawR+O;-Wty;yd+^fUj5nZ}mZ$Q7GeLnBv4^2)5`SY=ZH^Nx}=MY*=|_gthmslvH%Q_A0f+Xnje&ws6!V}+lWBP zn8=gY08Q~uIVSRZ!9wd0t%Ra;fg1YXB-dMxx4x;j9C#0wICz!^@Z z2uhq&;&qzORSR%On9o(oGtqob&XoS=**!A#!tvW~=31}k&;^HT?6cF5t2*BE zdW|{TRZ~d#1fdk3AMteZJjv6+vzrd)?YoXpuN=jTK82rgU1iyIk>4+q8s0$|$ok|K z(5(Y+n&IKr4|TW*77-Q+jQSL+4yKF>&O(A~O-V3q5m=8bs||v^Hp6VSCLm{Ip2WQ! zTez&9iF@|Y{L$B0M-X+F{g}lmz?-r&@yHP>Yj*}<98pbzSQj9NQNMvkTvt&^qZ$rj z4hn->k*B0#xW2ufmV^_m&YPuiwB~BFpA+N+bBUM_@E6$Dq??G(s{?l+PDUd6>V|zA z--9<*f%$W`BOqq*h8@Bz6QM@y7&bBFEC=E89-{%ZIQBEPzKs2=w)n!D{A6zrZ-FHi z%@AAC_n$*EjQV#4Yx}-t)ZeQDv%Kf{`%1j?{e7k0i~N1xH5z_TvK0=_F_qS%q>ny$AACzdBk+DL%atTWk4>9PD^@W)W9b*wvG!d={2?< zlF5?dz13XOTjM`9QNkQ3du6ndMngob=rpc4N&=(2Pkc@}@xeUdAatzEx_}D?-P=qGl1v;_X<4)HE+7Cq&-|D=a(t)5cx^ttM$09c z?d;rfpA#5Kx>5OyPOnY14%W#Udl3x}yR1FxWi&6=Z*x&_f{P`5%;Umt5i2+`PM>B; zj`Ca1O;d$r`0z+@^Ln=aIH0tSVJ6Ik6NX8?^^~`CrwFCNQ*HBam}B!9o-C`P82Qw> zbA%~y*ql3OX-r#Y&mQ=`c_Q|mmfT#MRxGE0o&!C9^&ha|7F;yb<|G=@r6+KoQ;J}eVf$Gq)H@x?tI#wLwSn;7cR*X@{ij09{MOyS&;ffwB=KGR* z{@QcE|5w}MrPp)9klW5Zd#K7u!LH<8{HFMZ@$Tq3%IET)qv}-S$`Dap| z#yN~mDMg=^vTMUYnfnG(O7%J=Vp?^-<(j4`1H_u~7zXQXzDb@$YhNLUU*QwPB9pE8 zY!D*}k|bsf2H{({VB~Bg$MD(Y5YM8^q`YZT9-c+{a8Il#*=Af!mM?TJdaV@0Fcc={WX4aqC!zk9k^H9V5B^>+Z_y9R@!o?^yuT3jR-Aw;|DFvwy@x*C^ zq#OGL%{A)xP^+;uq2k?QTN%%V(dTv1XPG=}^2e6&^o=FKH3Z5f<+afiSIcujWtf>b zB(A)?3|6pJ!+Oh?cvAUyOp=VYXvTi(m-o~1q>7U~edCDeC7`PKG@7DYo{=?amEqBR z9vn+0#Fv*_qC`UZ;cHQ5z)yJIAy4vE*Zcc!s!idh&olh`ynUX8*^A{Sj{lWUx9iFl zAaELIy+lrBNr|&+S$QSJiBr$tsq(7K^ap^LIpbmZy>?8&Uuw@UczONB1+Q(qsNm%d z7Zto#dm%I_daYIP+WL z1|}Cwc~FMpXU*Z(f+>$l@S*1L)^hpJS~Q8Bs3&*D!-6d-qoLGd#1u=EfhE|Z>!R_31KtBQG2#T+P8=~av@Y}VXs zq1r{e0tzSm3QHn>g*1^BRjlRK4eW~CVsL8*yU2A@$9cVvJj|fTnHFwMV=Z4P&6^#I zh5FsPmdHx$f3NG06qZ>&nF8)dbtWI)H1~spw$NC9D=V%4Ad$NeOZoEMikl&nT$rl$ z!UNSnDa?sPVx^~+SuQC(Fjms9c#@w;dKqHFEQGNAjQTdR1*Qms-;^p}r*n%~Wt)iA}gMz zEDngf^gRdpT>jb)*g_jxnjslSs*IdeI?Y0dG>N+7l%K<(jP1N9pg)JqB=scWs`;u2CH&wOA-Or~&HuLD-rCX|zo}*7 zL-^&|X>Z4gS80auZf@P^+RzKWG+mWiQG_5F(CK-wYvzj|$lejFt5yO7$KU>LnYpfTEJt4|^4@@y+=G zw-gj$rb+@(L_KdGv8lyYeQ`o0RY+=h*3w_h7B zXLlkT*chL6=!R&Q@L7MPwk~IwtRobY)tSTY>H>CKJX3hO=HDLM7+oLbyt2GU(idu* zrG&)atjNTJ+^DvdtbvqqpCJbQ)J{e?i*j)iNw4~nEa{)mg4hOm-v|MKy&MEI7jp5Nntjl?~k+#tWLfI#2G=oP})2 zKJo9yeqc^ZNkMyXW!!o$w#?Eju1t3OQmNduLCG+sUz8;#9Bw+_ShEqtIt=sW z!kq|BvQs~&%$l&8o*xiL-otj<)v>(|v69|!0zY{iJiPyjoRlyzG0K=VAVyIyXGPQ3 zs~RaBKe?pJUP>JIeHn7Zd5r`;MJ{0+cqf^qnR00TBXI?N#{Ez6#9H&1idfDQOJpCC z5&BO)G(MB8|@v@J}FUs^e%F4otidQ;@gsQvhHomvQDnk~>U2e^jhq*I& z!|?$&L+x$UsIQC_I&mQEvL5D&HR8)g0!K<2V*gSgRc5^mZAJ&}i%`q|Ir#>7O69^z zDoRmYq@w*G75C?Gl?25u!npjm*++}I<6-crHWT=rSGmCYrjG0lkK~@G_HUqyix@x7 zGtqv@lD*LZ4sUAx6ublLfDxB5M#GO~lzLE=i?0gF>f(>YZEy;pOq7|>DVQ~uuN$r# zG!5p)rol9}1)rM+*RMZ&8jM-_Mbn^D4&yD1rJ4gUqOv@3*pZIP64eXSS&)b0|6$Lt z=m%*N3P?u1!0}6`=CRMU>6JU9$b8`$WNsAHQph}P0GZhp$kc?P#c*A@KsXPFhfGc3 z3>-r)sWXP^`>m_1)a+tcR$<*EZXU|5Iv!@x0eA3*T?3f>)Xf8!EbHVhxS-6Aj(dHeDgxiHi{h#s$;P~-~7o2@j6&VL$+J(bw_S6XLQ25C6d?(4WKj$N3 zD;yWc=-_oC^dt6=Pp};A_H~Fr)B#2dTd&(WRLdcf;WvDz4rhm^FzU;R zUf~taYa~6B8s+>XqKaja*H_d@^rKRV{w8soRot>6agV9E<%8oMy_q*)p0=X-Oxfgr zbOrJ76(i}(Rb^_rk}e5aT1~U_eaCSb;iw8dPp9O?6;Zv^j?f^y?>?eR7+Xj7f}P&^ z`7@*@Z!u*@rbrOL0H@O9u#Oq}ODQ`ivX|fUgQbTTc$CKqj_Q#`7IBH9kj)}c>q5}e zmT3#;J|aD%+$_o?!PrQLu;9Guk4W!`M_xqY6Ds5b`J%0pxD;p+T-0NHl5N_6P}SlU znyzS0m;SA+FugG0Waqd(D+TAod+qH7{cD^By~bTD*!~*ZCt9N6w>s(L$3o{4v8k^P zfEahZ%FviQ^V)xfPMHmb%Iy=&+DT{J^;hAT=v>$$`z6VGc9IYD*d}2tg-iH92pik~ z7QZ@`JDrdtxW*Ak3T|`|WnVoKYgZ%uA!l%n6H8h~aHF%Qox_dI`+Z~kdv?SB2f|q8 zxb|dl4T^5JEp_=rNbcV@H{oH?BW#*eAA5M z@_JilaAR)m*5H~Hr5?(+YkPxhda*r0@T<$Fd4tN9w?Spg>pXMHik7R9;uba)CTA3~6Se1gBvVB)1t6|T3WXlTflJ7jPk>_FJ zoCGHEH(@Up>yw)1#x)klHO-Bu+{WT~f4iMyyp+CYpmbn8kXEINgVw_WmsGzPNVQp~ z@wdlG3wOrJ*T#Xbl<_<2e2BjX_`8?Cdj8h)=jHEa{>(Y|9Sh94{}@k;r^WL)&*MD1 zcy{qT$@3)7UY@-?PxCw-bOgaTo4w248wH0MkxqB;cbLB}Rd!a&((BtcOOH9<4TN>? zqcOB`2t)u!EN#$x+3Dc={L|4v$T~9!sh@q!m2%dWg&}$n={X3b9Rw*WHYR;~OgMsO zHB?z@OeFE<6@>5Q?`i&;=GrIhZ7jC?PblbpprH4MK}Ad3qG`^I)rolj2Wk`=2c_IR zC>=eZhtV-RVfTL^BWv7+a9?Cl4$5}YAQyib{AKc&&0j8mQ~CQgf53VSupYxRgC{Ki z`_g#A+`rGolWT(eQh28Dbn2-4DA=429o z^JQ{Y61DMqqHg2z7;06<&zfcYY*@*M8;RR^8*v-g61Q>V%HXqg)DnDAey{dD$}%$F z+}rZLsHvYDl)EV#S*dSh5_6wRaA};`MvC*B&c!KYqVdAM;;`M5`<+Ht;k<_38$;CA&iQorx>0pAr>sv6+uLoq zC3REty*>T5x^G?PONh*Ngh#~j?BVt`J?b+quY6m5)_H7EhtJMZE|l-;7(_h38?nHLxe~~aKyz{_b`?{ zD!HNUD6{0I-|ZHw6fj#x!mADXI?U6LKVI{=HR)CW?K>Q}%4xh%^mf3GL4eM{RSvn% z;2c(AoyZb^WznzE()_mT1FKU|*ja4tTB?UT<_qIIIKhLrXWXAI%8TUv9ik;SsOiN; z|N0ebHoLpCuTZ-gbX}q^UF-S^SEtwYWv)&%e$(;#`)$I`tUEgHbqj7{-oB_T5F;kW z;CmtyT7=5{s`GT$X|J$JQibhh<*bz%$LZ|N@5ZQ3%SfR*;qzf`L6E$|tR&K4IVHjB zUn7b0nxO-pcxyW+UamH8UePR_2eQnfApDZ(&-kcbcn9mLc>T01EcYM7(WGLC%wm6$^Rx{@hqGW zD0)Dg3y*6#niwd3aIlLq+>K2@kn?F$mRsM?pxpuAX_MqB%K3{~^kDa#I7gf>%X*i2 zs=dDvB@`zj!7fEmCtb!>&uH^R3E@;&zv!GA-YVWms&w-RZqt9tb z_)?-f{Cgo@Nzd6bnVAXC*;pHD0`n6C;{w++FDtQ##H%BK%+1>Dz}iWHwV8pnS%I|~ zfwf~KqcgBJE%ZZX4diLFXv9g|9lGtM4@RzlL%f2MmblBAP_#KvoEDlxVpKhf9f9KE zQbDljVK^wC&$L{zx{UfVQX5-~IXgWZ=S@n@;z$urNPU(^O`sV4$EP`$_-nV>q#@F> z@OOemjm+hzIpQsgmazv?ru}j?5_>#)?tGS54`nmg`FT@*4$IG9<>w9g*znu$}E5FuWdMrseO_ zolnH-%vN5ts-ab64>RkAqWG2aw1SU#=C@3Qz3I-7kDq%nq!URMEgE0va10p1b5duwS^HB)| zCn&>q)b`j`Pi?DJ`_NN+%Bc!gttLQ|fRrz(qEefx)g6b{s2GAG^ZxF&_fCS^^Yr=S zx!!ZV*UQDuUUTpD{c*2*-Ru51ZhUwbh*3MP-?J4Ucykaem1zDeRpIXQR=cXg&0dVW z#4dcnLEVB0Rk#pOf6IC&*Utd*9}|-+t&?jQ)=^bi-gprpbbQYU|7Mf zKNo#7-yu8zwjk&@7}+JQ6K0WThs4y5S)#9MH)9#$6)T94Tr-jFB5m zZnu$p|FY0?Um~+^-I;AooX-j8II>NC-bYgeHxpXc7!RcII?nKy8n zU>bYwYm%@lDi?A3l&D;zWw&yny@U?JeVyFb`C96))pwnBC5PXpM^4rsk=@z8YUDB< zd8}F~<5Z$&(^3hXxxn+lZ3LVrxp>u%^vxo;WTAOq87GqgeKXPek->UbAqkAaKg&F2 zw|(UpeJpggh+0Se`vS%rHpHtSZ2{wB8!|v3>RMA>jLO|6no#O)GhY48tIcrJ$Ig~V zpc6i&RbitL86c)Y0=YS%4vqvJ13!Jl*&WOp_$j#kqqmvZoor$L%txG^Y&cT}?wwm| zyw(HG>I;5Jz(vDI;BV8v;7g?9xZXTaLgdWNt?q9c_{B_kLW;)ZHX#dPVr=mpixeFV zolh9vt=y=G28*je;~uk#NUHkxk;lL<5czro!LHBneQfSRtf#btJv^S(rDqcqZKF8)iO?Zp+IlN~CQWhYygJ$7 zA?y^Pzwx*zB=jyY7HjmmPH+{X0WFcA948d078TC;pp`UV@7@>}b)NP9=!^s;Y9gBq zVi@DSG6#?m5gE8O<#h3vrVw20g4Ruv2T8au3te6se@+tEnE@JB6<(h>Sp#1shr1Wm zy2IVyU5%ADZ**Q0UN6Uymzh)e_mU4(c&YK(%{0rT6V;jE34Zt7Y-Je_48cuvQ>d)^ zC=AW&-9Ig)?wQmbY%hFMy$$@LJiZkpJsz$+;Sgajb*S;8DW|4gw;W}lOk4d1WJND! zA3qd>MLjId@5Nkb-x~yVog0!40uKf=tJ{TKn(!&}>EsW{l;k7vYETm1)Rgf~}! zoABT_yin~jo=r|pFv!I#J8zrXu$R{C$irM(Qm(%}WYr*?IGkCOB+VqURM<{?8ziX6?ph?I24Jg<;h zUMxvo4RV(9XO#vs79cQ&40(1-=xUy^JYJrGVIxn|HnP7u8k85(6S4DK{U?O`8aoF~ z$rwURC!|p^XQbkU-u$v$5RaxXu>XcJp^KPd7ff{!A#|>3lX2Em2Z=Gu_-U+=<6~mb zxga)msw0UQIa3`742H%`mE@t(#t4v5PXn*#huOe!$%yXmApxd*cOmQaLIcH%WWK?? zq@_~W;KwJuIm2=C;7I?l#wLB&nq(+dB6Kz9g$4<0JOMAG4u>mTbizd<`hawhrJ;V< zZ6-ZY0eAr@%ciI_cXl;OvM7~4`CtKVRQ&~$m;kG z!v_Ws3Lh*v?SGS4?<_SQkmOtwnC^H%ZOnp>y9Zm-|Tk%lPZjA*s_x~Giv5$|5xT6cjIACgNv*!dc+D36Pm)@zKFoYwioD219~o&Xt2IU{#U{=j}s+o+(_ zY8bU*KFOLM!cFdI6>@LfsDP&%RKjvU@@8PDJG$6Pv|_eEz|nsq|NO(z!pnNm^5ef&%~sTHno*o zI*N|<4?6Lwd0Z+FC8jrLw{TKC?rf1}pIp-0?EA<?IbmJ|Y;(pV`p`hchNOIEG$o*EJ;wJtE zl-6Qj;=y=Tm7S#sG7zY;mZHCb$@Z%7252K^w2Tv1YWUQxp``Gs%e9oM@ON1-1GZ*p zsV_SkIAsb&^KxDu+`RUblLtow0!cA#g5%55I*rn0tPDkUhGta1FSoqL_vQ%^$R3tc z-KhW#Hef{cX$4qe1J0>-&yyqzEP#e`e;z)d4QIH0Qd*~kk}<8jFs;R89gB9xsWaG! zSvg0Lr7(D6li)><^7R+aPTerQS-3gfNicN!a&yA%SXnZBJ&BA9;zC|q>}W9~gJB2^ zT)eG}y9oc2>N~?;i1pY8`~%(C~o7xEXB4menC#Atq_y(uJFy zW_8{)cfr`9C_dcr(p*}G37%+C=G;C;P#{J_8M#`z_U~qzr*QWExGfCuU9GWib@Ht0 z43UN@^BP~^$fIKg?UiW-nQE#uDjswTSRuSSv!YkkMDa~U9bF_Ph3Tmj&U^Gm+@cqv zn4TFH(=$UcJw-h|1FkApOQYQ1qMn{f?CGej*y`!&=#dE1(`_<66{}M*J>4)p(=C;f zUXJGiQ+Zv9_I887-DPr!F>+8jwG=`I(MS$)u~4y|RZ4%{8k|~eTv)=K?u=+BL@EpVLmpDvH6NJw1NSlgqdd2J{nBjx3%(GyB3S2lz~UTp=GNL;2Xm zN3%-IiD!_e+!ZJ`UXY5YgVLI^Fmlm1#V77CM$V*xhzONygCoOsaN-MJ;^?@S38_!t z;flC+6o#klaA=>ZqvdsVH9=Qt{3@w}@gU_%bY&l+TvaP}ljP*tbdcnmy4EJ1d?#9% z=xg!ZhXzc1D$jMEFq>375Oi6Y+(0HO2Tvm>PL5m_X#&QSbBU@N65@u%c+UP%YK-V_<)ssdG-zkh62|QCPZhL>}`BMFpGL>Qi^l=$`h}-B8)}5lK zkl5JbN|`b>fVt4le~0;7YaQk0Mw{QHNm zw^xYqU8t6qnKT<>)#`;2oO6vez29%)d#p&W@6>WMcj% zc3FJ2JAk@Qbpi(!F6N3EgQL=!Uh2@_7~7Mbm>N5C7V4F5v{m%f^&~RNeS0v29_(Oh z_VnNsdQi3!v5M5M*yze(8`Qf&^*_6nRv%?fyd4_kJ5zT~_Q|MgcqylRWAAje3Fy1j z-6&dPV}8TZyZK!;6ZKX$GX12`FCQd3Cn~Zf=k44DZtaR-s&Ow1O4N7c_n_kH#}EaI z79v#p3I{#rO1#iE=WSdnt6lO6e>lAPLwjEgH>EpUIkYTeY`F+FMp3A6gvk1bX=J@C zm{skN+O+(YI?Bz)`{BxCj$oSc11VXCFs-FS+_wPw0#$pZ*QP{j!(ZlkqDuRA%1?w? z5lQHmsYIl{U3V!+k_f6H<`jMuaMuj(QRas?z<8QS>HjHtafw3C4h0+>bePSXLG#*^ z%;wE-3unc+T-xS8Ms3cI^4V;TWEi^0{JoOqgKHdIz=_-Xegs{Wh!)-oQle&2zq0Z)bk~w$u^xw-cbWI;X zzf28J84wR$6P~*57$RdxnC{7L4rPcgwl;8X@I=f#H~4wXIXBo9AGA(*PA-);oSqCi zj14wU`enuo;blBeP&TQyc#q`(MV$>YerkR(_Vz@MAv(L}Vv9<<@S9Xxm$aPAe3u(K zZK0K1?@4Yf|D3W!&jDNp$|x=)yIZa7ZsvOPRP&cT)7u=9DJgMnveimizS@Q14o4{C z`jGp2ZD6T1v{V{e+S^c-E;13}&;>MfKyTp(WR;o?op0Bm4|^cooNkSuvzZwvGfu_o zdutElMKF?rbLT>ZgeNPy3t#aC)j`wT#+dMa;}RaVGwPV9bsm5zn(dZMI(0sW%q{Ime#vnx1i<>z2?&iXH6P zo&u3n(b+XU>pYjXd`pUF`_b6-TbzwenoHjs%V_>9N+3;G^Dn@@DQP|1htH6#b_JP4 zDw|-XWo}8C%}(@r^Lu8Bqcc#{uJXO9ohx@T1N92T4LqCV9x|3Z^l>LR0vdtFf>KP)Ka9-()Nj_1y$tLQ-0Rm*Ra zewMDPmajO75z|YQrmStAZ%iua>8+b7J4`@6a2^=$XkTki8*p{4Zj zg`VxsNQnvXY&wUiTbxz;r}R%pyNrgvdy9j9OCnC}rsfaKZs;T@++0+NHHgzyzVD>h zrt^2e>f|ctcQRs^6F#?gEPpdsj|4I}b^%T-PLn{2%?Nos+Z)~IQIaH<58Isu1S9?o zN;PYCJvF1qjM(j4oO2j}6U~*URnh(Yrwbs6xO*ddl`mhFoUcmG&xlO|a&GOF{LNfF zuF9D|ICe27lnl>~gUpeVjIQaa=Vja?;^vfOR{`H_l2eNesfcsaq3B2a5&Fm;(gh7D zpqXRZsf1W;A#{`K@HBoT9aWg_OUnEM1ThMp(Wycf6=wL7vMk75CM46BG|Yl5H6dBP zB(DW2GaaW-E2Xg6NpXsh!_j>BGTJM-7u0OMV-0&)AWQk zBv;=iZ;yToi>oiYN!%fZhJv`UF|HF%G+PfglUZX*krg}CfjYRr_v zV(&mghF_F?;aROsR51&P$;6jE#DmR1;*}ZatJnv?V~`2DN)QMR5+fv{v;3pk}pt-@!G66RL<`|B#-=@q!aIhVFX)ib{5FBEIZvq_N9k3F>j6D$uX4xOF>P_Pf z1Oqns!d~#GKya=NPU!`Y2?Wo_4zh5;j85*GdP^7^2%e`BKuNvM7pve)1HqeZ@GHIG z>_D)@25;>Jj|&8qDHb8ezIRskmj!}lc7oNt3C0J4H`(CZdcl_mf*b?rP4cXeLKjB{W>?&_0X35Amqxaxk;}v#0cMBQuiIlIG4Axi18O zOpvHq6}s%OJ5uPj!xJKf6YTKBNZ~~4rVQgHMm+kzHIN&Mg9u+H>&=~>=0v?<4Tyah zK{oV3@n@ts<0={uuUlhW#v3w?3<}mD)AFLEj;JWTLT4s&3UNB~CCL&Ly!81bg$G=| zq;w1No(XaLk}@pFArmsemo&+OykbHo`jVztkY5YrY&?MNJV65f&fKtC{6liX3K@7< z(tZmGOb6$SL6$&iMC&%La>^4 z2iRI-INZuWDTS*{g+0o9zf1^_i8ZUI0q&iU69eR9eHo*^$b=j%wXR{*0Y?4oQrp=*8>)ySc&yNhfb> z&Lj}*Ggt0fch*EcGyPr)d|akXB>8cfC(P%^Wp*%^AD217Jbqke0&}=yF{Dq= z`P z_c&v!dQUK}QtyezW$Hc57_Huuj0@CziZO_Hcz0fWRi2TmK+}yx^`2>bah25QM&ncU z2Fa-R&BnXxJB%RtM_fjUiDsL>{Rb+;|29zX8cyYR~rAJ-XY`X>b=JJclBOp ze4n>eK}lB|jRM`S*6vyrlxqy5Mg{Ma;9?bgP=fPRP-KaWA{BgCg40w`&XLA7D)^WL z$El$31&oVSaH|9dtKjnzOjE&^Bn- zp9(UKCB5AKN94Eel}R^Mf}!*z`2fz5?I^NCKJ*HiiH{8tZ>IwKPo)BoZ_4KKK5p!b zja^*HDJhYArP|RMu0Ze#Ec9>^=@t^~tLX@3&4DtT8@y7$0ux!yv`%4g1aoZIpMcSQ z0?r5oC)(hbt*DT1SJ|-tuwc+`SKF{30xJt%Q%riZj7=&(h`Hzt=(S7jcr{j9h_!4R zc8djrQX6N(3N0Aq)nzv9%HAy5CdS*~p;lBjh|6tQiUk`M2%?f;*66bWv(=DP({0#a zfT=AeIsDahCOD%Afsm5u={$mgoe7_25VN-Kp;5FhApyS zHwS_@*svlCHZKso(T3&rW_fEMSY(4oSW#~a1ZUf@0TygYAUMZ{eSyrQj6-!GILU^6 z0L*ImvOqA`1|P7Zt_)z5-0HU%Em$ZJ^x3fgWx>`2f>UhRzx8IhF2FsAm3*BU_1*?% zZXkHA4X(7ng1ZC3>um6B3oLj%5X_THDWNQ{>B;+ELF<8FzMbF#E5U;bJk16tTi}Nj zxWEQ~oUgj#UP1kV;20Zx6tLMFYEcG)W9=_5TL~Uh35MHXu|!Xo-YYvqAn3Kh4_aW^ zEdtff4A!UB3#)odcv^kzXMcp=7fbY#o9rys>J_SfngxDIfz8zmINJg@DX_VK0cZA> zARCaqhWYXZroJS5*^KOE4EQ|@EW47uk^%3tz=CN4tY~Qf;O78aU35f!F_$!7erzR> z&C6cZfEz9FF$FdkHsIwJxLtwGwGFtmH+$LR?B&guDOLj6@9Y&0c(esRsKDkT2Ry(6 z%RXqYbHFFB>#2RG`eH71zWkXl(m5v-*j(*^TP(0_m-d1O{2L2=T7k_q5BTBU>>VIs zd)f14y_GoyNw$7vdBsxd%cYs0MD|(vhUld0dS5578*cMgEcXS^=6-^zL=f9A`moD1K|5C zu#gruY5=TRU?DU5pa#Cow@ScZf_-1q#}1BIE-4k40AI6jGptMu+<1lai;@dcB44*} zbCeq`VGxV)^z}Dv{1Rgo8&+coW@wKgmJ6>}PsYbPw*Vc+hUskNYt-W`o-FGVVvSkgu& zM5JgRcc9gR$Ku;%BP%v4(e+R&aGu6L0!DLE$)u22rI?FNC2e0d@^U>Xa(YvU?Ouf9 z6V_iL7H!7q^+%&cA8$#3>3o4_``kDz>6p94HQ77OvpvCgaQz94uukG2|7Us|u{|k6 zsuD0Oq#c#3{=;9X86e#wdRq!3$Fe-Yl6clv@#OhqPk_(=OO_Jz_M6+gpryQ$tyQ&d8bC7^0 zV}$UT>ln78W5EsRLrPue8jmZ;`+Xn}2;~0FB$J-p6Ds673Ti(=o2uUak|%PHfXy~U?s-s!CCiA)T~vQm zuBbLd?vea-bW;jLv5OI_G_I9wwRYob^*&~dSMT?Yi+Sg~O(_L&?WmE(uX3afV$;ob z$i2JOh{#W+a5!Z@m8P_+;@6bVASCpcV}XRE(1wMC3Pbb}Gw`$$kiUC04K`Olm-xa| z6N}-P7tcPR4a)vhOotD^WU+#!56(Ui`uoBKoA#_t;;9PH(m^)p8EYIjy-W z_neRjKlG^c4E5~Dz{R&H^^~}Ak=~=#^Ks5$F#?mDp}oE^yLrK(?)fS!%+v2gvw77y zu(Tz!etC>LG7q(%nk6X-ghX+`E&j(#XNCrOc3dg98~v+%S8z7lnMi!>IP9R>{m56P z+IV~$&F)1!($wfC(Sr+?1UAm*{*qBWa*muG(LG*&srfBn6k#AOFJS$O-VBbEpKCu1CLbuFwMBE8bA55SWx>Gb)xXSG?8!L?? zhwt?*fD(K29?_Hcv)N|eA0vT4-sI%!n-fF8b=@8flcwS>F@c^%ku1M9nZLKJ>KAVM znbIqBPK&*(@3cBvCSz|$%jwx2d2S8OH-7&d*f=lxu$-{<8#1Gt)&V{FW+{VpnS=XKmcd&|t|fq0$EIF97ND~Y6#lfGxuR`MfXm0dtDz)&)J zk*!H0H4-Q_{@u>Z(OA;Vo74ch}<1wajSsIcP- zYSC5TPSkwiyO4XjFkYA5o7u0>Y@h?Fa&q_vcVg@rVkT(kV0Von5`Kh$6Jn3*1@7O8 zI>5+g#Eu*9Ry~nnW^?k*>IQxf-81q)U}Lk?o~}5C$T&Dax+1*TEy5nL`MssSH{NQ( z6q+EDAt#=DS4QYY`9*E3a$l6p}Zc{U3HvN&_J2PaQt?no@{)mXZ*&WFj6}t0c z`DWAr)BTOhtiBjgX8iCuX>D2Wg8eA~v$Yq6S?zjZPm5{rLRrA8y>^4`CERkPcL{@D z%9Js!xz(99hSu{!+Nx}t$Vjc3UCcGj%&6;{N3wUK*N%0rFYkOc zJyMj89yE8!@d-xFIF|208jngN-^DLTlgru0M?_Rp*Cs{l;s|8jmEb$G{@MBHfgfIo z)TN1;V5XHB+y9G_<#N>=Z*!0$v+)7phV0-zB^s7>5V=3(Pbuzzg*h?sFwwim_M{ zq#Fm#bjMCHmaLrhsM3l?TQn?-hi;2AwN3t_r|~#d+IYr5;sYm1DlF0;o<=#3bNP?# zh*!xjS}takxiUOeieNJ04!!i+eLPk$?O6kxEV*f_%vH9t+ChQYit zCG=Ko97B>^P54j^DDGzs}q>eKl2!ojjzU9Z9;Hy z5W+ZuPWy#DuE=Wi(_MOsjl5(jE3)QS!_2kpl9f_w_@!!{@$D+JyZ(!wwzl11LSFT% z%h>_%*+w~>oaCIu>g;m35N7w9l0eK@7a5q76+APU48FFNUjj%t)b?)9MjpPy+2)SjKvm!=`Fh$U}y7 zaCie6RL*Z-G^@_*vuM7)O`ZP@r4LMzwYnI%yVA8^9{?Ei7 zrmuCidt>vPD15{Qh{XhcCvR*%#cu@Z!p#SpKX5rQ8L7Os>@(%GdoG7VH!14iMvFRl z`csQK@LANsk1c?9QO|a-2!r@aYa!KFKlWvqAW`vd5iF@3A)N-qj} z;B1UETi93%8t2on8KXq~FN~|(G2;2BD5;Ns=RWj_E{2Ze93aa3 zB71P$_^)3=@uiyxgdIj1fLc35v9pXH@C~u^$Y$9^kfTc9Skx&8G-XjTdQSthEs_(x z%Y_Pc3JYc0oOLr3K4vBQTF7Y!Yn5Ig+KnMV6B;1TY zKu~-Ymuo8@PbaHlvQiYp+g-9XgBwCcD*3q>Z|TWENQc;PmBDPuD^o=|MW+ld!+NU+2_)QDP}F@vJ?3e-Yz?Gq}MPhKGEldzBV zsWSY0g=kP@h-CQ&`AmQWQM6e=5g($yD`1m|(4_&KSaw|_YZ8-TiiqghlZOmaYfxB# zr8#?Eevi6FT?2&3EWrS5cIF=@$*di?=`aN3N+BR;CiwQRKL(G(+zAyQ_C+(9Pm*;W zIuk63PyBqIhL!`OC(rjFyMbh}K(bh$;QMm@L9)OpgX{BKi3m-j_SMHv{0)tur1mDt z9GbId?p&B?XK^~oF@CUO*EgYt+QuRG9;J^|UMxf^M?cJ0?A*}JQ$;{NR3u9VQDM!s zscS>vwQa#$U^u^$lG-^HbqH0)!*-&_-F;Gsu(6rDxgGi^*^%Cqu%%+3q^$eQ7txqt(eco}pD^XCU| zxh4CQND4a%d3LxwRlZ%d@VR!bMyS(^Y zG+OA!!;r=9i2C8dZn;yk%FWOrpWS zd|;|unpID#u`|w$QW}Ep;9z4Iy7T2-NiR`}#Lz{-AqLWwf`^lWDdm40%X%8*aO4G@ z#+jXJrz13Bnz@ksdS5URiVicUj2k8#?YfxU>=?!Y1X{#=qUgHH?UL|!ZC3-IVRk@V z4g8ZErpqn_ugeWnTkj3iN9WKJ$~yZqU%z4cxnLn#h{k-%G!LNOvNBx7Yp${~@i8FF zE*KZhux`^8j$AOszAqhpmUn^So*ul9HW~dVs+kGYya=$WMVR@Gt0hK> zltMs=MLIl}()`dlBDb2UXrNG%@=Ay1hq}R&OF1dyoe31|?+iaLry#}?>IkNJvS&M% zz!UI~r7Ua8)(a$HY!>F?A~DKJIu8r&`@-@In0rdTQ3sxRwILmsL0vdFviaT_D<%8MuEN{*LGmDMh<7-BeoJa^p2F&-Uig4Wva1 zcgyr^MkO|RUWSjBe&`GANU%MZPj&D$bp2Fic;ZrCiH^`nUYw8u4SO0klZ9?-h;5mE z#i&sm(-SuHj!pcQZ+E9B#@=oC>jp^^8)!!`wdER?Oc=#Am|I8>Mo0>+*md%)p@%oV z>u7=ox?5_ac9XMiM(G2H3eS%GRO~Hbee+ zV9N?3-$CHRhN|!U)vtaPOq}2G z3w}S|2Es!BGFtzBq~zFBGr5Bc;kL5XxfllirF4BKp){)0|WjSQ%|LaPal3u}{n z{7B=}IaII6eSCUKv0R@AkBqOy9?LA_+R)bS#c zN2euVx$PV}Mc*Hta&LB9PSa<azYPxWmm*S=ClD_7ro{XzYJ^jFY} zkBfQPtwAS^U!dRolTJ*+?tJZ|w37CbG`^^fUIBP*b${VzFtCXfEMy5wQ`5tpcZjE+^KXP5a_CvjB9~XA2Qs!_TIcqqB!{=o;Q!81M2kC!v zFu(t8gPEcRGqy^v->0_*hd-jml7Za#p*fOs!Ux)C$Z%vI0M)+sq5f*5{+N>?Y4@RR z-)@cJu{yazGzV}D19*`+fY#`>+oRVLY3gXdj(YSA+>f&N+5^{rALC@VRR(}= zJ*%xapV4={_96XU(SGs`Jm{8ZzbfVe0pGl-Q-4)Do$eahO3z1brYDPz$%JfYzUC3n z*Rt*nT{}wWqYB!dD4n14cDZl=I=9*JeR}<)j-#CBeEplKk>|djOK+;$F+*~iScG~r z$6KHcJ(Gp}&#Ehpjuf>8y{~;J<0JK@HEM29NmWB63SMhxcAPagw0-|l_4KQbP_vx3 zomWW>3p#!vHB_*Tl`yU18=Rk*6*gy1L~Bzs`s*F4dQ$(4p?-5-_AdHSYDs54XLY8` z;8wF2ffdq~;WJKVFcJxLqSl6zb8kN$__lu3>zibTCdtrZTEhR*Yahsk1`HubLt5wK z2LY#HDzZCI!@1cBm|T&R7#6C$T4LlHh(Fsj^Zm&^8_oZV7`L6;{{I;Fzr(o4WSGIY z(o_FG#?}7EW8C){r~g?P_b4^8F|O3^e-6eyPHhC^N)2s{E3iHoS1SBJ1>>r^d=uju zRPkTIxWf{9Fz&|V4WR2jTn16{Zh$P0_#ei8Z_e9!n2vu)h}KBOgJscrgaV(9R6G`` zcrsG)Xr$ulT>mHxja;78b22{%nG#OI?i4nyk_>%sq-figRLr7>JPoJ%g$SjFk{V9+ z*AP+pa7*g6%|*XbMt|>uR>SE>c5wlAfDcS@uTI8)@T}OMaV&=u!-huL109pyT2e4+ zUaZyga&y?oRveE3T=ID~-A{61aQ-@&_&WzJnina0F!mUtJJdQxLm&|zQ~9pi`t=-L z@J60)IF%jBZa8&W=;DS`6GG25oGS3#KbbrmPWjiTH*_p%IF;bJcPxNla)(|gRA#by1 zMl7L_ejx_5zxK3_PXJ88oYe{FAF8agJ&jTas_kin4HNJ1^T#Y8;(wCtIYbU2BaYhs zr?;feluSb>BmSrPBsCaq6+cLdcU#5JwhI5iE}X9MG|Ii4D#xh;1?#bN@AusQ4=8P` zckU4u->i;)s*REtRba$D_a>4e=<2wf&(UdTBK~dW_@)H=bqob&4dn=!R~c$34+9Lk zI|j=u1-Ukc@}UlqaSWz(FmKHUJ{ohH^$c}4bX?JZrr&rI|CapFk$zqKC+DUdMMO-_&`t!riH!P0RTan<@V3sr;j{;W9KickWO1 zysZ5y+;Y(Q$;LB*OowAbe{TomjPlTl7)i2qT>+3D1>bpMkQ%GQRN74>{y$sm+Z%{)=r!d zm}g~#iv5o%=NKao;wy3_di=+tt|6WsoeCJy9)dOM313eMRbtL2kHGU^y&}IlJ@;NI zzh1dc{X)*5VqeEK5)q-7QH3Y{`vGQvJsV8YcSTE%E{yo^i_||HseckzlJlb4?&!oqq{EpCSUgq>>QTK2 zw71!_!{PZ22jC~2vrhYFo#yac^kf|_>hc&v|K`Z7(=d0ORq@2|@<(wJl{jZpe8cbc zqHTKp!@B=I#3A$Mm?Z)KzOFr_mmJ&JlGsFV*B^HXuHCM{v`iqR|iVD}R=%Z_ZA_Yb&r}k5i1J%6`Se&dx5J zeTNbXIGkYYv3)p$b!hpve*SH`e_QikhadX*&_`*f;>_8SW0XSn+WsJxpeSH|=-euX zGCrd!t(6`6x-2Q>1{}GXnDH-b+ONg*7;kJ3KjGc{hi+l zKP*!57WVtFjTk9;r!ba|`=E&botTR?_ZG*ISPFsHrMpU=R?~vPuN{ssZABZl_1SoK zBfNwcU2t=fqk*CH?~6?)@7L3`$H<67{n3cOB~rAvDx&R=dBr}s|FLf*AvbcO2wi(t zY^n@o>2s3w>?i`*qHBAbjf~ipI5>V)-^J$lT6AGTY^18r%TjVMF_vL}ils@Epo`zH z&DZN2()vEGaz@F2XXhBDR<4#%DA`-S5!ExpAb7m+Zob7Pko zcYO5~Mu3fncv%*0Rv{Jw)4bz(aC9C{Q@@-vFCpY=ad_p&KT3ns7vSFFA;kDMQcupL->iQy~d;lnc0GwD_lP@OefdzWt9Jcpb&p1Beps3%-N_|BH3 zU;@Hu3``4?5RIW-SSI$j31ipOco8uW$a)Pa?C4Av7sLE@?iWYIUF-c~imY#WPSaJ$ zmV6h+H2dS4ECI8`5x8OErM-Lea4b+yqof(I+F)xj4ch3dNfq(X6;a&yyeblLE!Y8( zGj2+&-wpC}_=?>1>6_0dWxgbhwPQz`i$e5$PfOQ{)q46v-1}m!@dTpwgpSL#)}SYa z+uUigT5&xON7Hb3f5r7imzD;P**NnhKVoyj@x;*0`c;n#Yc{T35QP7CltkQ8m{!aA zF`XZJg%L}V0pxb;X#91sNX1cDyYbgip&eknMqZ{>G`V_Rs95U7Mg%nxF;N;mp4W$X z6X{?kfLP!xEqnR-U%;95zmL&O2<;#Y$>Vy$QxQBoRW!jkd`kF|CHr8FRS`TIDQSCB zKeZr&?;q_52yCeSPQchqqw;XN+Bo#CQ)DFslAV?_(&SNgIV{ zrZBwfQ9ac){`Q5`TzXVyT{lQrU99_$idBz4IMj^`C8dVD2WwRw9oBbaAccnOk2=bu zQ<6h;=TRd;`qtSkHvey6&gf44Gtv!%TZ&-jNvYy0sbaEK#bZ?Q@%R5E&N8*jj9veB zCjNx=Yq(zVK6QcDf&5Ns7psvH>}~N;j)G`n18*bVU2UDb`f(4L3L8mIS0edNzOKv z*4f-JX=_-J*ZyYzzBPF~MWP?$9ODImTd;Dd7ul$$otSJtv~Zp&PMH zi04@C80nk`f(dhtswT7ZkE_nVmUhvi#?R?Fnq&No1#He4<2HifSFLgA5Opr{z)Cy; zN9;Ls&=5*s7{iks`W2e9Fm^|L0!A=foVbQ5HNJYm8Yj;tC8gjukT_jQ*A+V9W*V|? zCIs{ZJNo|K=vh*hik?q&J9bO5w@^(Q%9H^DO%}>9=o4gk|D9UFmd(aXhS_HXN{vPR zIj_K+-JQ+{XJ-uO-^byk#&ak&f$!zLg+4(B^uz&rf(nLrxytEb?SLW*rb-W^rFE$t zU0YMVn>7Mx%;j>aCh}&LY#{WwMAWEp&?2ZOeXP zTPu@O16P7&_O0Z!V17B(d`B+#b0AatUha2OO$?DG3z%x|?5So8?uz4~$q=ofwpfm5 zhm&EO3u^iiN1h@YCW-KA4CpW*mNa{z@_ZbZwsnX8=-KfGpzMP<)5HI?vgg*v0k!Ey zZEp0&QSNLU8G4+}L-(XLmv;m|`6}K6O|hXj1C`6lROutP@e>YN_Jo=Xo#a+Dm=Ith zOE%?qA|%AGx&id>QqEg~ew$PA=U>1#Rwr0ZD1ryH{Z{+JZpOChAc4!&-BRlcbBfnbxNB8af^14V#1C(}|NJu2N z6eM02bMe>9O&c>nn=D4sS{bU`dbekjkYG7Yn{RL;wXMz}a%}iP)0wtW{axP81&Qc9 zwHCN0I6988_sH2{(|u}`(?#8+p(>KbT^@c6TujUXA_tzQrBt_p2Vyg@uiv2DL|~a- zOGD3QLiIuU(4Xj|`FT<{Rn^P#i)$tBuZmnt1T=##( zfeGS&gTrx!|4Cf3IUE;hPjcjTwnpk7o6Y3sbb-M;U4w*gj8x!8=tQJ~i>GHj;dvDH zJ4sQMTl5%*Wly*eNR_YtLC?KY%+S+v)H6E#@(b+jopF*KMNV+_&uYk~X&xcDRO3JD zxJ)92>mPGygJ;+WJ)-p-^tPJ^y%P%}N;4^ytMsH4S$1Td-OFc{zP9rG|KfVGuSMSAPWqdA8`HQfC9 zx6y~w^!moelFm@V3@k{Bp;iusO^22@_47B8w;T%paVVDdNxW#Aj2V|WTBmkEEMPWm zlG@G>d8+i5s;j|8p|I$S8P=ij3w0>$g3ecm!cXZ}>qOXHCn-NotACP$+vRe}|1L8+QuG0nx++rt zQS2gh@O$#>2R}m&a}i>yxv$qh#I2QRtaXVXPto(SWYWvh@EfnlO+KeUwkprvKVxR) zGXq#!a?$mc-ko!bscd=%l?h${(kn8LM{;-EJ@P=r-{?fbt6SBUUas=BncNKB--uAy zqn>b%OjR|{g{er$n&-mg5FY{m?L0q6MQYN{V=)}H7ejiu{)huzH*vW290%5-6G6lS zw#~!WdK27?`gV>p(J6@fJ{YNg8Vyj-jug*t4(j<#>_?+hGJM_4=`VV?7>`EQT3n0- zy&e|RgW|*5NF?cOa$q+99#(|z--=k?>2clX5zy094EldtPhZYJ#J8Dk#v`GpS$dozxLcW;8QWP3oa)Qor=Ale){F#fkVn z-T%B^pNT+K=<0{%djEN_z~zD;6U^?*^*8ixmHFbdcyqq9CEqn+b9qw7)6fDW2@aa) z>t2t*16di(A7^a*_zY_yrzz-;)IUnyKH|W-5R z+IxCEcLOh(i;N)z`&@JKFKiMF$NJ0kikI}ghyDS!fsxRTPuoKufnyHNAt@P9-@lTU zP0?JUnlvd7rnRV|)jz7UCo$DL_qUN3cw65at=~-*o^l?3>M1UyxwWQJ;z!Q^Qgk9h zFKM3u(Dypys-|fN@u%zGjUV0gNi1gH0cL$$-B9FB^jCd5P1zKYZrHR3(+Ne-hY|{f zIl=Pyf{5L9358CMea(8w^ZEhuGloY>I7+uiGp|cKNF}1KtEqJwr@O64);AkjbZbRS z8P$J!CiSKjEXFjj3p;VgwSw_!Uj3t6@O;>jIOVE!{#d_;++5Awk}%gt)auYd8X6z@}AANCP2OA^&Bg1_|QC7G~`auk|bpp&pxKZ|90e4CUhhU zQ~hnz^U2~Lo97WIq?{)4F_rXu104D_j+is?+wmSuW1AOJ8t6un%>`$$BTqsxYLy6 zTsU&?g83Uy5ljeWsdpj?_G;%SFP#;;!fkmKyZYxf^vGzX|M<;%4Ey>|=j_>(QxF7ct5VyS5dVv?5DZTvtWd_AcRklYd+>I+>2_IN=$Vbe~t92Z>Obe+Sa`{e5* zKS4I66IVu!qMuW0Tt|-*znmTW&E?%R15&9wr^{@`0@aHBW-Ioat=ONaTCv}3#eQi8 zBFFmPk*|g>rK_ak!+4t&e^nd2alfkg0NxZu75A&kzpE;rK@)+}05Vby;GM`I*tO>; zDsxlc$3+3YHY3vP4~IM5PS1aW4E-l^_B1xFmG1tfWTSfgm#PZV>AzH!kbb8lq~Cuj zCEH!EAAC_ppZ>m-N_bVZzi{O7mi``W^|$n_S#8zl!MxEj>hp0wH}tY5FlHz{dzXq# z^9Ep1C$a=T2+lC8!XyxU!Bb|%Rf3Y3Elz#cP zro5t>^2WC`<&A&V6xEP_tsTv2`*T`;Nw-^_hgWiZ&Fg)vcCd-!Szf`K{R0fj zK%~+@YMCW$8Rus~z^YmDgd~M`@_x=Ajiv8?74O*Ja7K={Ff;{?BF&p&-1>VNN>kTC z02Uji^MIVQvlJV}1mRu#Y~D(ys)JPEw3wC_aevtBqJWcKZ2UD1jW5m{ml)m;glmAw z0*7(E3W}Jlk^Mv5`TRvfx8DaAKg#j{0;rt@fs+b8^!u_{55z=izsQR&)-?^ z(pzJTjqQJv+Kr$%Ns|(r<7-(DFGW0pble>3Nom;s8^Z8^tUp1 zAw{><79gHXJiLqd>@gppHu7k9Xz zE2k>xxHZZ*O!;Ks3=L=jqT}$Xvp*TSlkWFy28B8{;8MlY_zIzFS!(!88T34ib3~iP zVG13_aGYoS0r$@qXY&Pux=8T~AEMgPz+^VozQ%Nrvj`4^HZ`p~XM>x=2f5WE=*vr1 z4zuJ$nek@?xH-H_Wd=QxE%8-V){NZH2FK}w*iPdTvJX#6($b8%s!SPo7^(d#xYz0| zGdxlf6Ym6U7Q3RiQqdU4)D1Q3&@rm377mQeNZLQBuOB#U(_uqA>!;*E^IW77=E}*V zGao_0eeD7sCazn0`u`1Jex;NVWL_rQOSEs=jd}HT7UIHHH2P;)xsYqR_RX94v!=y97zlBO7#7u_9ILppgy%TQ!Yqt ze&>f~@Bb5}Dp7Yj?zM=TD!n*;-AvT5WLAFaS=r>M@~k|Fkk{xq_0 ze7vX1qqXr6H^%#oW>(F_aC1iN5Nr!Hr*Qa*))zozh%u5xAI==v92=pS+B-v8vR*ty ziv8U45*(V-&bHVaEG<}-46UvCWKI1ti(iRn4!A6yMkVj9SpP$emcR5YhOH0Siy=h+ znM`)2p|&9V5PmmALv4Y{YgZa-E#Kg^n;L4Td$`!miE~6l&EmJ48fuE&9vj$q3CYgE z#9e65$>W*DV3smgp{MZ}-xSv!S1Ln$tNPXbR#lb4uxzsN&|lPQJ8rab*UHsOuIb`$ zE~_m2FTwmNe#DA?3DFPNXyWU2$CSD(!E{(WOE)AMvlVpp7BOfrz8jyn7 z(P{qXYGww%X$J&2OAzinmd*M!pX&zLe_gU5yD?^O`&Y5)E?dPX#13+XiyqVSth(00 zQzbT{=!m>(170xto$TUIh=!19ZEqOV20mAwpPk1U0DjEr$iPUkD^l#vElKy>Bjz00 zL`yOvMer56@f&~+2u!PHb=HRY3Ynts#|Ho>?oK&+b1R>AN{+buorj%WK+-xRxD|Py zGw`WMoy)m1au=3ea9<+FTc7`ZQ^McLe7R6>!Q{)z1mr7{!eDKYNG1Js;B&c)PSr!FLSAy zfs12^y=Z!(x9!}>U21iGDFf+Er0q&5PqJB55eT*ar?L0IYgrMJxvCY|$e+94ceLHTlXeA{|jz) zJhJx1!p|lQPoC{lM_sE~E_HgbS9T$zNw>Byl{_T8=>lsr$g*fk3`&m?17#Nzu7Z$DkDl(cbm*G<#rHJ zjKp<+4tI~*a1?sTvm-b99WM7DK4nEvqgL zo)?|*GbwG=h0IF&I{c5st!`~0JCT#l&w0D!5o!=TubgPy#u9P0GyIV^ymJpDL<--b zy1igs4%M^{+GDVmgFazR+oZ6f|UhtbOl=s(x0I>|W>}qjOiOY=-p0erLm+y|AZj z*vGWk%JF+P?Cl}-eJQ)^YQS`043HIhnsoRf1l8=FXVB06zEht=Gy{oA7Vl;v7vvEG1xgl zVh#*9PlzS`?tInPWb+^LNmSPN#Yb0z8sekZ@b~{a|7xB~am@Z*iX-@6DUQ2&?&tXh z&vu>`p0{|8^EiH+;z;MYkY_y4G@eqPWjyP6w(va4b2WKg%<~rE<2>Cwu4hslSv(VY zZsb|QQ_HiS=YF1_@;uG6ljjJ}M?8*aDUauTo^d?a^33L`=2^pYKhI-4+jw5)d7a1L zIl+@kn?~_W=DC^YZl3S+Jj=6>$KXlcPCIz6=9$G)%Cm&0mS-K$gFHXy`7O_Gp8Y&; z@qED3&6B<(#W9R$0#6~&tvo@Vbv!@hd6MV1JZ(Jh@;IJPab)m};+euTi)RT>i02-j zhk3U0yv%ci=L4Q@o>cnsd>;At0(~j}0x}+7``30$zGM7<C+q+ zEtuwb{aAia%rCE+=J;Hn=JZPH@)e>5=w6=0_ zXhrQ>?-LQ7Xw7O$-fRW852rfP-b&dS=F%4IoMN^P3=>&jNFzIr0{TueRB>dV^7MWM>CclPC^?H!Za%H=Co zeIt%^ym#>mZP^lU&5Drs_Db)f#gcpF5{E;5TfL}GeW_ZZ)d0({@p`qIJ8M>~uJKmZ z)~=}aj$Yzjv3RjoTe-x$x>|an7IrJY8#Wa<#+{$%xR3XUp?Qvnc_&_w z=XjF$7~T&z7C1&We3t=OT+OPKH!DyUpcJrhY2DQmZ(kac@X|W-J9#c%&6xAq526EQh0jy4NnfqdMdrQ!v(hMQ-lI%UFw(yfToW1KfoQgGQ# zOIJu93k&81%5S=U#>|@?j+y@a=?WIO$zPOzgMVSE|EA*W=gbz^*?E%PmE?Ivj(66g zx)n9vKxOTnOKa-9i)xm5L)Dd_i^}DT?v(e6ASh1CvGeRx-f9!~I2O(+b_DWoo~g>t zFSuEXT$q2;jABVQCm_Ei=rFNbougP`w>q!%(PZ!FI-#T&t~85WxO~yln(@mt!6|l; zcHWXlNe>>bqIB;!lKH$^U8VPOt0?aZg+*Dsl{Ma_HQptaw`+HJm#+<4 z)>c+k)(SS4b}w16SX*9M!#IH{y<@-tw`)s5A0%Z4s`!$a-yq{fOU90ONbN`0UH*-F zNL^KFyqTYiD{o)4_)fxSMU>xmBMasf73I%X;|JJLrPVA}l~_?XzH&|FV%9;`n8nMN zjPs5kKYnZ~A@5YLIbp_E2~G%^U$vT*6f|a0?Hyysrq))5wAvbPZx#|~Y%23Eb=iu= z>;yfftXx!ESGjQ6l7)3E7A{^C$oDY$@67dMlpvAmgJ@ zh;(<|n7$c|O_g3*IL^CDdS&U#Mf8Imp-%OTH3%TfWtHOvA&wEqIM&wJ26PqkK?c5Z zSzTqS8J$^NSsStvNJm&PBo>b~dwK_`&ykghnuzBoDJwz|EC4tHz9w^-rfu&3m(~UbOU1 z?OQXXPUKm)!pu?v-GMuc%w|-{ik!E#+86nBi|NWqSFI z(-(eQQC26JeekbV`RiGEnH?iWeA^`X#`ur8oE`7-WlQLf@v^u_j8*f;>TUHJKi)fH z@(2)&ca=F+m;`6d6sykWP>r=mN~JX`Vh@knZdtl&p?{-@%W3Y_6DMD;)zzB8%O|U# ztRE9FLY4om344-R0XvbI#9A&@%R_w@j8$#bUUTuuoTcZ_#-Ai&;h#OBYb)!tWrBH_ z$Wk41OCKZYWsQZZ6)v!1{XguzX?Gh(k~TV@`77$cRugO!AVtY)Tedtck(8~sweX6R z?LL%-2LwS976`BbP@=};|9+n*mRza|3*}|}-rJgK6IGQN85tQ{Mn-0`x3CkUbM2K# zoXJyAwtxlJZU+7AHnKgMa6n;BrTb}l2vHxJ+kiJ6!u`SMeQyjxL$UTx{7J=FdwO*5 z?xX(nyU}EN@Zqv2`g)~t{lz`JZ>+C0`W^k-xj4}umzQ1pLCF3v0Xy;i$}cN9RyNQS z93thORujyXDtT&Y>60Qww}@5HQ(_*}gVy2Kk^*@7VtaRg|HVsq72DrEIoRDkc=d8O zssx8)LR$tix|&{IO-HBAMz4E5Dn9-0#s0y;KldEKN^CHe#M}Zb(z;+H7(Xk*KTc(r zx^vKW3fd+NS0cJF9=a>F#d>+28B|KN?`ennnlF@ef&3>Uu^CNewDsbmGkLpSyzdOB zVn&27`R=|rrs8Kd8|@9(X#4~sYYCXAJ1Nr~!!d%|x9`?6;aq}4t&&>dTYpt+=pSX! zV|Dv=-^{O<3I;fqq%2V?(5%j>oAymQ;t#?tZR!|Cr;Xft!;tnEdVMLE#? zpQFi%nvudsUbyfhNe($BjU0L)wc@9=0F%z374;yiTZvrwXmS-bPB4w5YKUo--C>*2 zU#CMYd-7voVDAgyPMR`(&~BKGQWxGVr^rS0H93!<175FfL^in~>)#i<@F@2l%het9kHwoelJtq#|G%uag@qm*3{%P|v8$aky$n*w%KFBrc1*<52 z#^wDp3QAj2{cUbzLDUq5*VPHi!#@G*Yu17E(8Su^LLa00-=VcOj6m-vHa&0O~j zHPrM#j3WptuucUx(9n!`5GJ;NnKVxEsWuL^*)P5+Zi{-vPp}f(7Nc%0Ll$)o-v`wa zjL@Aq2#parkHqMn-xhQfQkaT&&$q(st@SOjVe$^<3-h7Wkp+a=t-)~jhK#Yt7R%kA zOMC5NxqoxIR~_n<)Q20dU%yfJtUSurlEx!aYF^IhU4POg7dzXkaEAWMv;wBSOWoL> z$bT<5>kj2eU-R%nEdv((X;(TD%D@zLib&w38HyvQf!``JIp+bn%gbfZQ~Xd^FzjAm z>)#A(_Tb`=M*bXX5#FUKg+MjsCa3PX;CXI;tP|(x#2-j#DCp7ExZ7KT+T-pYhu9e5 zRoIcs^|6Tem2;2D;pCUC|QP)e^~Ln%-P+Ot{h#1V3vQf-Tca`Ew* z2&`mhI>p+eM0WT?d%|D$&!{sS&BI%t-D=-z-TJRvpWbTTTDx`Y%^JD#)~#E&p=qps zrP{5(VxEX%uQxa67qdcJb6^P!3Mp;*tOikNi?Ha9v7Flw?Vsp4HLuaWS}dq{eK|mc zzbPS9p$5i%08ONGtCm8BV65I#=E_9oP{^jj?k2ABdz+E+x=yF;#?c z49AJax2+*GJ|ZfD0fEzt$R3Wz@#?$^=df9s)0#3`0v;rtMZisPRw5cvv1R@@x&I@CZU zhSr+z)+F*HnEFDEwfN55!TMe(`XnR&mo>YvPyGv31wz>49?*nY>6}6xt|OOV`o7nL zQc&&>ln%l88NIP^okT`7F5-}!0W98Z5HV}90@8udPWqXMNzk7E5&W!d5{TCBJNZ*u zktWUQF#==4h!YVmB|7;Rb0S7)Gs-hYZN-zIr&TX2s1ke|5E`!(iyVV6UL}Mly^G7~ zhb9z&4W7ixAWuLM6Wh8@ijAKib4Zs8Jl^4qr?6|fpN_8zbk-dm9}sC)f4zV?w27eW zm%6^Z?R}V#P|kIHhIai~Z3{xQ`UuLs zT^Guwnov7@`1TF@Vhc}X56Qr3Hu^&_R{un~6Q2kmrxB1yTk{k6RfW0jDWA8b}Vg{UTd>LQoRI<(r6=O^;H8a2=`7ID^f80 z72Q(6l`xW+AS`XbAyb<#zk-k4@q zrd){suEWy^6i(zP?Tb#}9AOr6h9Idfk=-*rZC*@}!6O<8qsz8P=Fz}H5lWG&%1e9TLmr6Ot>uM_h0+{eb+RKt!~n9r~f)-|4~C z1>tLG{l4Fy*WMhsKOT>+@7&v*-IZ%Ca5M0psv@K75iA$-lMw?SXi`{D#I=?SxMT=xBfY<&(VwodM~(lPx)^rZT;+)i5cvm2v*ETm(~+ofLp}kI^R@D2NHs+jRHZf%v>b}N?X{x(SeKap8 zz^E7wdC|<(wV0Ju6|17m4KB?CsLDT)ch!~gK|torg95{LfdLk8gYGu{kPEoE5(?Lc z_R$e5=Crw(+Ur0F>#&cbw?*41^Jmx4n87mTGFRLBvOPeKxqV4!A{nY}E7CbRX?8*v z`p9%2p7~18X}_l_oR&ZLb0bNxXOswOw`XUtlfQ(CIn8qCP%6H$9$qL=;32O=2Zb8? z%}Sc%YCU2A0g1JR}ACFC-b(Abh*m>7FA9rGW{g84vx7j(Pf~IT^r(7Vm~@pyQup)}iUU$Sy<5 zwgbG>^kJ!xfVS5WvX=v_Xx`ZXh}&sf`l}bWN9aA?NIfRb+y8*ksdQlkgwbnvhkrcmT5*xF4IJ%dzt>VMEP9*>dU$M*NRa8 z;-@p2>A9I2xR|Aai(Cf-~4JOT{3Mi)hlwEDd2reisdqG{1y~u+kzV*t4VY5U3~2&kE?5B(SJKA{3M`ButYz z&zJNDN`2=5ptiayGJ^*~6J6R%Y`B9% zB_9!I&$7yt>4DWRsi-sCehdK^RU5#rQ11;&XL zxHL|OE|E@_z*hi*uc7ZVC_XZ!1Y?zaCMG3RkU3_p3(ZrAicu_p378V*Fo1l+2FMKh zhBd%=;X(4>xo}~N)Zv5o)V7nvHaMvQ1voyT5v(IqQ5;r_rRbUgD$rvdB zlHR&66EuS~o*{+FC40S=FFh#2fgqufle;a$Awym&s%c~CwQO_`O-FrqY~Eg7duDZCVfv-zt5pPmO04s4_mfaory2eTk5X1}09El-x5^sxQtL%Oa)5Rvk?mjR z@PPQhQ$%-#BM^&cFs>&e{`1OcNLWD7du$|6+~=$@xLP(#Xm-|*!;GM#T=I2sY}6fs z&8YZ(@qD*hyI;Zxz=!Ck#tg=z5mu4=!~V428JzSkx!FWT8Y$1f8F7-469);pb=^>S zR@d30KT*o6#VMRE1zVIk9b)z;z{p|6N_ZB+pi#l3@Zz6j)Ik#8rjDUQI>zSQE&yTK zR6hv3%HfPV4ygW{8ty5SUBi`rP1zC(_T56VQa`40Kq zwNa2zN}uSGI0N(!@GtsKn~<`h6;Buw{oC}ROSo4HDr+{u%gWp-KMQ#yon{m=A0qV^ zJkEfY(?Gw!o;zm=g~W4^dA*VqfenMF((lHj9NXXw0j?3lAnu6-m@yUrP$+^Ntt6I) z7wbS+V`mQHP*--rsyfORF*1fE?^L1+MHW^7BN6wg0^xv(0lqGjnP_@r( z#(_^Xg_NdG8&ct$%7U$-B6gAD2~4DA8gK7{BIKMmc}GXdoJ?@fqjA0Y~F7ODXN&cp)3*${A1=(TZ zDl$;NtmaIGc>F;@a9|C}ksPd`)SMhn(No0c@wqKLB!v_=?-y{CPm~5&Q5L0L2+n6< z8mCe~EilHU>xn1O!%bO+k!kvxOlig3d0<2Ftx1kabDpd+Jd(^*YAp<$aXwBmLh_z^ z!ZzkwM5iAbMzC0g<`}R3Wp%wM8h`0rTz-Xxu(}crE%C3b(R4lh z2ntWbu~hvm1>u`n%3%VM6EK)AncGraGarZGG6Jv%HLNg{OaYaAr26WX5YH;sd|zT^ zLrG;PO3Ja2wfXyit!d8@f*|8b=eqh&4E%o;Ofo$wDw-;YDq41&(G#YpK?IB#j2^L* z7+EVMx<-?48vM$MMTxOkKhQsj01N+&nY1^ilnhWq@@J3%VA6z|B#Nxd4VwuXVg|s9 zD8>I`0p3Be`@uH-gvD&v(kM5>3k!r{W8P>qUF$rMXmXBKC{@`&!x0mCRFnl#M359^ z#Z^<5L=c=1k~i!GllUAts^oIJ~fELPhcyJR9D1V!wkH{LrE%! zYK((RRyGHUAek<}*Z?{bETo6N!zQdGhbojM~rhLxg-MF=Q-Vu3By=CHbv z;STRYHd~8Y3UPT&%V#}#kJTBFPMPHxg)n?SW3sNaz-yuno}nnPkUokeyB;)3AgfTo zXg{?+yQyh&k)FUmye3F*i}h;ChSzD?Fu2{$ki=-}lnbFqB)Io1p+|qWrKjAHzAu@5 zPjRY|FnHxNPetqUb8g6Vjf!JI4Qr}KwqypQC|d^eTD1ioJ4S^vU?-#LFqvh~g_oBN zAkDTduM3cVdSSy1PFC&Q=_n8Drt6}=-Slt$k@Dd>q#w6PB|;h+d2p;=E{K_EaqU@D!j`8T2M;(?M6PiR_t=**G;sf3cW8)(Ze-`MSwLVKz4ue(@9p*W$fiY^OFJT=8CSwNwp@H!_)*YvF zq;gJmp6^zsAyu*A(Tnzrz@o1MI!S}>qXmtgj|Vk|^ojb{7_i#7?;hzj2@_LJ?ElrKIMP)g zS}hy`kwi(s)lQyrlq-Hny&GcVB9=%^j=LNcFH;C*KIa>;f@;4YAkKh*HXV%y0w^!h zz|_5et!xv_SyC-fRFOy+)3UHa_8;@bxNe_Ifngoqs{iY6lg3GxIs{ZlvUNCoa`1)t zF##`if2GG*D-dfSx?Fz>M#qyAQR2xf%zZlJJ`O?@&hZaL1AKdB)KFuhDn>jC{+wxi z6S0H>nxPjkBHmO&sI`?~(MZ3Ne-QdCU7=hxq&yjtBae-LXOs5x(US|N;K6TLkuDkD zjs4(G2%nLYql1IibOgkv)}WT#+6YU@mOzTb9u$z$PqTu#9MX`Wbb}PE>65g?{sf(# z&_Q~nkN}4pMnzF#IJ6NW!WA3w2Q`xoIU9W2c*A)q;VK_VNqghUh3^ay5R;Xk91z}? z;jkd(%T#Elw4$ZP6#E7Pojc1+ii+!4ja>t{WFWs8+ zZ?rq+4wyTY(U|*&bWBnTe*}*VPHTANX81vnLk}P`L4x&VS@x_b#ZQE3#RMo9T76lr zYei94f329L<#q8=1Ayh~;R6K-4<^B3uFFL^eGs`9&ZAm#7PLmKh<#SgWxWEQ=7kwH zV@p;CY^rBJ5iB13y;V}K z4qmJ#G*^`HTuYVPb7Z1NRxaV*S*fAWjY>^Nh3gf;uQNA9fz0&7g6FiH@%J5K?QU1) zD|g%ulkXbpR2yb#_%T3H_OIA_mibP+EP1`uM70=@LB6WCmltlIb-S5U22SFk^lhEs z2XkKW&D?y#w@g;euv?8P)6JbPwb^Tvmn=x-Gmuivi_SIJz+T=*pbXn)S{lC6Cs{1O z7TW<%edl5F3_r<9GLt+=DJaN^zX&M#A_MY#t#lHq^!idtM}@1Cj)NS<%FeZ`NBM}$ ziNZ+0Go`9LEQk8&8Lm8<$4^qH`tW0btZ&Jg<;-{DXd^`e>rvMVP?Dk|GCU`UiF#1& zEA~mK>Ruk-m5)`-0BHe0+0r?V_LrBR0CRX9Xg?vYAsN!nSdLlCIQ|^P(}H?}ksd^( z%K=u3t4(4G4PBw64?f>19%+we6mh1K&tkk4odgIh3MUlWsqQ&;l=QgbAZO;-Bk-0) zZFE!sX4Hx&_= zjotrnaa=ZPv3QvkD<&!Q1j&)GVlWB_3^FbcghfLW@w&Om3OgnA0vWL3Lwb4=NTDRa zhU&EIm~zKfo}!f*om?L4Qo};`z^m0!$lv$~l-`p1!gj(g|KYrF%xh;-Zm}2XcjKes zNUqtE6CIAhTAJgi%5!}M3%2h^IMbV+7j?usJbAgyCV3N<2}2p@QNV@{!=a4$f^!S# zcOZ`wXY5K=l*03}Z+ksphd37wd5}{MY-Sx>Cg0okX0KXd5f~hLNAGaw1>!ZT_J-rV#fj1Y^8D!&yxc*xrVVvn{j1fLTTD|^LSfkDC)|A z;7KX7bIxPz(<~)pBPDy6xQC0whLtK-(Zvsam6<_Gr-#0+4ZdZbHZ~-oL=MywE zYx>gbi8p1#615}>c>#tK9YbTi9OulcqoQ;V3su%BhFCb>X{DM*>4Q24`VADMt%_El zZs+8utBEeUHeFPan|Ta-TvLV#-dI?eU=@`#?ZaVDs3{*f{E`RGjV;w!TxPT%w2E`FB?o9e_6zX#I zv_6;%yam@&0(&4mX4jqw2g+$;AGy0>DQ9GHe@(9;%S2OX{4WB7R!VxNx?8e7joSEzGY>^3jKA>GIxP}|g{wrRqfE$D5Mr!V>!OVK8bnS; zQYV>(Vs*Gm!Crmyz`q8ZCdqEsOd7u`8*x-@gKXlhxdB8HAs_~o9=;3zl$gCc8Vov@*uM!9+n~En)_5tZf;~h8@KgU%MZ3Sw zY+r1j!T@=qDR~^vuG#5Ip@ffD#WR;-2^TL#G<@({t?79mr%ebCA`nV=&L!xSkS|fp z*t?~|;XAPsYcF+=lmgPNilI|_p5XpUHlsyxmga+ZvXx;0s#-kJ5H8Yd=!1}<+qo!k zAj(K?&y^0rFFfy!gPC!Ckax&gAq{WNQyw$%OnQ}|X^jRf1w+9qCv@hY&>d4Gy5VT8%-Ek|g%gmTV1G^adG z#J5?hFoCHSG#h6a4D5u7Yr-my7d#RElv5yr?}<>c+@;IduBBfBXs#C9;AGVFIEsg; z;#+2@U`a5E{riFj)#9mw-tvA3F~x zF<((-lhrYgtD^aLUrkNL#(x(@Y3MBI53 zgYc7_-18K3LQ$59jnv~oA9EfAH;Hjz99_1iy)=)}+(c_UkinXTO*?rt>FtewQp-k! z%^+dZ$8&IP688XQ4wDwKDGbWehiD4^mhum{S%tsc`6NtG9UFD+3?laA(96H^3(o?e zbD%R~GBzm1x{4~CWkEFkVwmOzHXT2A{(L|+07i7M0ty>T`ExWiYNwsBQHv7yj* zV1-CPP>P|nNGdeFJDiR=@%tHo*h!YL0b0i^Hp5p+YQB-?4fWV8zc^dH7!C;0ANlDQ zw&`q|A>=5qKo)lp{Dpcyaq=1{xyF(ynF6sQJMLwUR0_#BvDKmoObR5-t`HMUo&^rv?O45J)WHACauQ1$Q)@j@^#BC%K*Sws$ptnbUf~d2 z^b%`kIGax4+PEN7nzz;!!U7^2gVE@1VWX|XX>nzg`m136SMQm~Nj*2otPIB*b2F)d2gN`esNMCpQ=ah`9FI z;$zyIFkEDT{(8EST8p+!Dl(^0Ny^?iG2YNQ;v6(=Yy`n|#N6bQCoQHRBjR&_5D-IL zD!82LyhBo$s|ljArt957Ue55 zRx>GPsOPyPjz${)fgC6*?vF%2Dq^IyWHb?X)&$jkl9sS{f!xR5SUm;}<)z#uj)1xWVfZOO(aPPBlog`^ z;;xfvKP{9iW~&0#U&Zc10oiB=%gD_E-Lb~ zrN+v-K9$Mhg;LdmzM3Bv|4}jE4wT51vldZ_aR}CfcmW1JgaiU212TXW2+H+IQc={2 z3^O)?GHTcDP2|WTsT7GI?6FkOOqnh*>j8J0Ldv4e1OS9p7cRhD$;kM?p2qWS&O0}X z@zn}NbC%I@V0qt{@4uo1OE`MhOF|*+6&7ZCizkC13&&RQGrgS2bp$ONoCN6FS5R~h zVaGAbVjXKM{5ZPRagVy*ioHQm{LT8Z}YSl{Q6nYM#*o1Gx&4?DK&?UjqP?J<{2*VZ> zZEi(Lqhj?(W!9hpbo4zBAZf{8O}L~S@RNmQ|Af5|`SpaN)usD#&Vn`V#dR}U_Suu= zWsG2B`H{$Ff^&v()hjr)&9v@d_YGI#RUbxSGvbA@)etwV!~Eh zt#{oWT%Ev?E@f}q`-Cf5+urH|R#H_jonx2mrKY%f2^E>rnmlMBx+X7_AMz>KT1TSr zhkg)gIVY8pysV>!cGVrj?T0)#LJhT2!h7o*c4($4$kG zyO0yx>>Spc3=%6#Vp7aVN1xeMk7`+hg&q2eTp>d^H?xFeLY$ggj)6NC^OFSwBJND3 z8G~$G40{}-aC9uki1>=UXLPW8-&p|l^MPk`Sz(L--bMKFJZSa7v+Xdv091qpup2U& zUtd84U2Y_7KtXYZ2e1^cyeO2`#3|-@gOIe`qr;U=oP!`Ym*O>k zUVh-;U0jBH!XMAE*w9${h1a)AJKWxc2)-a25R@2KT{Eba&G#16^VuXCMornthBMfb zN+8C(wxoun$2b}4si!pXCUE5^_ch+;01Z1=U*i{YcdrneySmQ}p}x z`pQo4`08vIL$m{~f5wZ}OYffyvHpRmgOuJKa&>fnf4B6wWzW~hp826O9zrfaL&ATI znd6}JYB1_dA9o}I&k}sxhb$q*K&m|J3^C`V!NJ#OkGJ)Q=I#Bk4TESeM)&SM>QA@N zd)#g(g`bahPx{k`!;@`v2B#ERU-0_6i>pFZvIzD0=<2M15|_Y2=ND$xB&wkqatWRs}8 z+TS%e8aD}psgEzYPg-&*fSvCam3p|IxvMwK;LEGwq;qN%!l5zGpz??il#vVSCdiRWVf^^+*7g~QtRJFRRP^rYej5l*ToG_o#H@fb&*Vs?tkoKu7qqt2;F#XX5b23inRUR>)fv~MAs$i{&x^^_ zVJ%%sRp~awZn3##YQdl3@lrQ#&CZCqNj<$i2}CJb6koWiGwN0K4#T9Lz1z?QJN{QkxwCJZ&8tHA6Hu>#EONGuEpS;V=x~n0%kOkKh7l$`b4swG+hfvU`4xRSB9t?ct9-``v#Y~g9(NxYB^F~;!e3UKF zsyDR-TI=SvK5-Oum+?Tas4Z+L70<5fkeCHhc=qKcJ(zg+)vO|o2D$ZL?4te7Sx*X=8DPt-I@6OSC{rjNWQL>m9>aub%AaF*>OcxQ{Esdt+=B zAu%}mOrgt0zpU_5RV0TXCFQ|gP{{FP&-G0 zN@h=Z&HQ3#K2J!oPFfkCn@WO(2BXsJd;%nJTpTZPz;U7ozz4@Xr|<$_TzOqDKi1#a z8;r|&gxG>gIu!aA!{A-PI#-Bb>!y)rIL05NpfnYHF^dHW!aGS$(s#i;8RDS~1j@m0 zUyXTb>2?4qsLd}4GikZcf;Na&YA=kH3{?Q8OY0W(v*fGr5jch4VRveP&t$IfS!FKt zPFo9K0SMkvAgC&li^*A38s1kUi||y5i1rE90Iq=xuJwSLkRZWDJ@1I0{m?8?VG#_l z0yAi4w!u|O+4g_~Is7Ojqq}iYoCk*9<7Ogsl>%MiB%I>Y;l`O}m&1biB~u7!#%!m4 zB6y~Golr>P7&8juOalkgcv3(m`q*n&-%)KeZsNb|1rJ&SRwTA)ZZDaf7+{q`Uy@H~ zoa+dLptyzJogk(%>5c8k44y(Dx5dh!l50fvl=H+ha~)At1XdDiAi^|wdXp+!mm!mq zWuh$5W9+oGAnPXyX$)n8SLe}sR7Z$-ssfS0|JczKZp58_{(A9w=KR-ztf#7-SF?a) zq!h=(cl5!1!=a=$Tw#znImo9ZHytQBf}_0NA)%8DHV^8k9W#(gl#!8&D%0lfL0QQs z^C$l5Y?46_y)}yGEm4@W@l#qrjG1*;os{Xo2tz3|G@cBB13WOngq@}Ry>A&*h)^oV zOh(5MP&{iQ=~5u5a48rg!(1S)7dR%-td5KF9HpS>E0$p-lX~za=W<7A>twVXGht;? z3UbacyIF?1f1h?Gu0FeAcsD}ES^6`aK^26otRm|4W~L047ZcSGwPsc>)r_J|LX+ti z=uB0a8S`VUln0ooGE5r`VMOWWSGE~R)AzV~t{3e=3R2JyIL96sPL|VMwJtKBz5xp8P^o2^{AtToII4kf)Dw?A z%mNwgNfl}pxv=0l3tBW2%Ybv;KJU=7DiOR^v$hFy(CBE2%p?myZbg-&#f>@oHXJwS zddnxS)^NQ9IvaW=&Z6Q3ttq)0;-FLA%2O@H;yf@`PSilq5@kIVOUhN9`uR>B!|tP?V>-vS3(2C@LUI-**|_l8f85%ehyS)MY8isYB3x1#Kc7iqC9gT4A8Ce@>50Q^el zY1M-IS#>x4^?A(zP6oFM4b6az)%Qs&1mckjIX8^94>fpatmGovOa9S2DddYK3;Rc$ z*)`z|qWk^?;W|N#k=poA3NU(WpW~sxQbxHubO`To29S>GS8+x_vY|gglgP@T-4#q` zqz0iTGrtA26BFvJ8%0f{Q|f0r8)zlpfyoW$Yzr_5bry&;!L?kqq>jt2Cmc?46aNL+ za!#0I%b}IARHvrgL>Sn>ZF33>J^>2=E=947sOEQ6BTn$YO=HS_CP0c?9)`X5S*!?3 zz@MpMWZ%wY06L$HdK2ysI`81TW;SrG+s;0R` z_GANub(d#u)Dst^oT-px;gfq4D8h3=^iS*bh~moKZqE0Q4PdxE!b1WJyAG6 zp?7gP#c4EWgV8Y#sm;uu3NuYfIws%(Nzmd zi;W=P%)%hyIPVPIg4m(C*3?GsCct4?3^>vHmGlQN&1-IU=)@=k=$kflDc`b{FkUFH zsQ^v}^~73HF#&Et?LE{6WH5|FOqtT&L{9Pjq?-Gc>aUrcbP zG%+h0&-&a?GCG|WQm@&abOZJmqv1$O;k;w)RKmF@&+w*bUUVlYgMZq|O9xgo_Hf^& zOxlP8VG-C8edTPu+SA8Rp4JS|OYTeNC+@ZG6nn}@g%09ZSO7Zha0vqHZ`GLWZhLUm z!P4toa-a7GmsM`q=}(ZoL(2nr$&T;^z($j5Z)Wh@w>_l(W8#Gg9S(w$o#q{!uz7ll zV_;zCut@-4nFgyz7XogKjQva=<5j90O8gT>d$hl^UOewj7gFM!4BW6-yx1>(beK>^a&;?;H1_=(S7i=A z1CgI1uW4Q&?s{D;2A({%G(ow=O|xcj#!?q2a$=(&_XIa#Jts(c0E=QR<@kf$+_qMw z==Gf~rznYfXRCGG$EhFMq3e5h(~tM=rXO*lE;rV722!NqbSjVut?#i0vjxu)PV$V3 z()6r~Kts+Ba$G`lf>UiRZ6>RAIk}VeU=RE)d#d4C-YJ@law2yKyw@27O>liYy1sJ{ z>A=cnKjINdJezf;7icvV(;D%R07gX{@5=#iDitS4wuk!*C)tJi&%ocD5F&(BxD@>RCk0IVhG+ZXwCB{}=lb-^f? zIt=e4*|5~qJyGhABe@PhI4bl}LaLpV(f#F~@y^+~?9WO-NkYc9ZDnEtgn$F7+FidQ z+PtZ-;mIS|wV-qhR0aqOoY$6y;-I7yb|1b(7UI>o8?V}}g(=&sGLW4~2Ov{yJCndJA&owVbltfa7`mv`43uR^rm6Jf&qCc)DC)M>Ft^XUNDYqd6_lhM z=5>lpSuTo~k;c^<+Ui7TbZ9$WeInhaFRGT-D*78uVAuUEM29li#WaL+;sDx8)GHLvjvYH%k*sm4jD60YiKoasA~Af*gF9Y9}9 z8QBu0G*KzhRdxu9`c3(9z*1WGdSZ3ihCypA8Un_QBkkGWuu28~gTa;*{Zfy2lgp)r zs-aGr$X}|2PqXXAB>z&ar*hcnhd{Kre-b`4$dM{nyy4BQ{D+aE&mHuxe7e5&Pe}gH zK4J5ged>(;Ztt#8RT3TJ{xUDtx|mF;W5buN>fEt}RVDD>>v4r{Kjz;m1R zRZAO5FX(2i3{LDNsr+=~YBJuyb`kkRSoiKO2aC?v3|R6Bu!8?rfrzRZW^`GL{+%!G z6)pLB{l&c_AfgYKm)-cQ0=uT~i=YD_ilhkEbh;Ku8862McbID*RbDyBpzh`6Rcr8W ztA)8Spn+qO!Oy>$#FpwodQr`XT=^;t`Vam7PLEd?BwV4*k}-nc35Ker7W(;$nG!{fz_N0S|**@N)m?7PBk7^*Cu3r&n{avb0`heTSM=g6iIuw zjL}syQChmxinF^eElIyi{qkwBO#L!z94%x`uf08}_`N6FdB^AZ0p4>{%{T75)mzHv zu(nK!YMUYl!BHcJv+qKswogT5$r@i1ig78mVat#GmJ1p^oGn&uE>wMBxoMpx68ZjXJ? zSY?_hDa*}Y8Wm}uUJVAEmhsbSV|6xpQl~F4t_drLC?G6DID`HdUP{|huh&%vl(zpY zA9_;@&I|kPzw-+jOWuf!WxMEXg+WfXED;SSV|~?5NeXUDZ%Sg@vDgi`h6tN*=M~~m zqwxpsNJ5TzE#|AOr!rti&mR6`w~e^8r2r}Emhk^w7SaBkKHRhm9(lx02uv`o1K{84 z++Jv+6VNy#vq|fW9>2_I(3wI<7rb%xOg`t&gc@$S{JA05-p)z z43YWv4T^bbP^PW_99mv`mniAqPsy*go?tsFT|g-J%oD7VPL@A{UY!if`WSDL_#B?1 z)|5~6@i~6Fxzhacq~AGvK!sEjW&07Aq)+AJlG+NM8PaN=k9bU=pV)+{I2V<`7`l$E zYfSskx5J`(yH3CT*lj(igs|AhlA8NN10)Q7h}v~}q{z70%aE)_2>7|8YD>ZEN{GPc zU!lF3F@h?)RTQ^l@T)X;t#+P;p0pI^@iXU(wJ>iwSNGoi5qjL(T0)WM^G|mQ=1uf` zqHPn~n-Xo}lVjA+8~`Juc5vClUOj zB5P4+qwePsqV0d=L1O%Ci5V8#lqAC58 zBma!8yh2@XS|tyi;RcY&j5-LOrx@0uNcT&I##)Uhdvk~^Z%Z~a=%R@xb!n9pI7hdE z=HwI&uKxS_JV=X4TuNF}R7Zh3nKFWMx<_MHhv_oJ6hnwn%m&$bkFX>CXONo!ZbE7) zFZfYP4T+r~v&c&_gC%V6M#EI@m0@Z>3E{&SCBY_c_mKDX8f7{f4Q`@NoLe)N z79dKfknF*B3vCz-!xFY#mzkgle&U#doW~7_9VFzLD~EiIn^5P8ibN@KetxOWZ#Z%r zdSC=lkzsqF5xy6D2Txu+-&YiMJewbS(>0S7F014yAylKdVo5KIm1);7PC8$#j26WB zJ+o|Oj~Y(GJ#9AjEP3}9`rgrtUM#r}mp4jYEc4-tqrI2A42d6ozx#3@2s0U60E7?Q zFZTZV^2xW~9fVJfTa)4z(i#e{gkLqK5PI;-!(^A^eb=^ki^Gu~N{S1xb>Gb)j<3Gr zy6pt#iiK*)Pa|jY20y9D6;i5j^o?k895V%$QUBBP*0G^a8gJVR`<$mOT9M?X>~pi% z8u=Es!%>l%4$srr^cPj5GC^A0Hsuw}xTm49RAUp2q#^J@`b217iLD%+-fsvLrz$4c zw_3M9x52RHd0I)xx?d|&w6i8>t+kZV1N}mK3|ykcyVjzR0o-b`d${#RBiU3iUI(L; zK0e$Qsi+R4@j+seO^MKa=rB4+qVP9E!3u@G+jB~Sto@|n$6y}pjqt?RBm*5A)q1_+ zd^)|nzp()efvdB}{n6C~4oe&PcN^w3Sl8P1O&RbulOGPeCGy zi-|?SK>+M;_wtCSM3+QM;QKbd-@pF|a*V4H^9*B_L5Yy+pB8ohMDzbq5ey3-ON_X{ zCUO8Ca`v#8m@drXF*#Nj5Ag~QM9vjhaM*wafng-%pj@W3YY8!3-rS0;f=E#;D6%m2 zu}G>uPTVFQ;)Md1t9WrjAx*laC-4P zf^J#03B1)Ny2n@qsRND=0c;mO{bKy3lw3^0ewBNW16zb>0xLr`cBQU_-mWHUz=%b} z6{P?MY5QP&lkZe6|t6%@;^~IpDg!cz>*f;bxMg{Wvpz-S9aqEi) zZx|e&bOz8V4;mkEwR!QM-ymN=s*{J6*^k2T9#MR)#O?LfuYd4yFQ)zJ05D*Xzrpdv zKIKy5h#sYu(fO-(mZMfuFOsq3@}H3K*!0EXaSusJ*lWRlh6{LX%3;SEujwP*ZdXH8 z$k-8$P4UZ;$56BL)f@ul0gMQ)xfIt}yabT`4qDpjL0Mv&>gqDIRBW&%p~fWK6cO#5$;E9d7Okqk`hcJpi$X?Y6F$^6G3jZrD{3r@P z4_fSm44l6)dy$+N9nuv7kFWxAErVKV?tj1>sJTKg<`6T1fTexmIe% z0>^s~nbRSA35?9JZ{d*%aPlBWtLl8g#`}f1V+9fir&pc9V{G&7jRkfx-mu6a$wb=_ zIJR()hJ-8n6NKhvZ-XEo8aI07?Rg&m_Z_tM!;|MbFL(DFY!c+b+#tg2mVt4HlaSXU z36ky4v)Svql{K@y?Jeju(qkGDl5amB@lp#y5112l7`KJsp@EcUF6wR@6+^L%Y9IHI zXehdl0%rqYm{RnW3^~2t_xfk&(+7>s{3~~)smJ=D(Yh|WpMBQSW6YDD8ux6FeO%Be zFm7UNWTY(%$-M7I6D&9!!30lf(dE6+$KS)U%RJQZ76@2!HVM9qo}%Pv+B?!O^xZUT ztd^4*34n2sE7L$Lm2AD^2YS(-C;trz|Aa*YzT(#RPjJs&dJ=C|M6{iOl0TSYtRkU= z>9d8bTfUQN1E51}089FX9e;8YOsMgfPp&r~ezhty64PCuqmw!s2=fKk`lx$xGC4Z# zLFCm=36~I#@Bk^ZQ6`O1jr_5ZQd07hY+L{8{%AbLUPBzd#4V&3Bgy;47+^=DB>noH zM9BmYDC}Y-LwJH4bCI$qK)fx@$qQEA)bk>)fkDzxiz#BR=9c*eqHAXUxAPKAah83A z$555u>%P3=qVv-Tng+sk(I4{Hy(maDIf)@Mc`><+?u81s_lN41YRI&0UI|L$=mOMx z(AbL2(BS35o+?@ohG4nd**z>FCS-&*#9>w8i=g0c4dz$LnlQXe&3l0#M{Coktb~ek zd1LD~!vl$MhtG%cSO3kM`l|soCQHtRQxwYMP$#u;V3vXgo!x%K>x9VDL0i+6o9LG#t7Sm81v65N=dK5n!6`ky<@xw6VYL%nT)pR6E zH*7MDuhbtZ0hfJ-rEFN}FVHyXAZA6lrc7F=s|%%ggwLt{Ex{K7MG?R>rIUcrqmz%) ze$*nV!-;fztJWHw*68wCH3&(?;Iq!-olm9>$X`T3^Ufplm)Ep(FRwKhSsI4o?MlCipsD% zr|@!?WSUBM%rJ`ctR4Y7$+z}JLJz?wsSg8SELaS{dqbY=fH;fpyJIL*d-dD9F&JR7 zsTg_AzQhgBCpbkIP1gbm{-L56mC)t1+j)o3O9{B-?k^@C+c5@L<6Sc{MW7b=PB9ta zR1|muOxs%`>dn%#W8P2oYqFYQapXMS&0=6aS&RSk#=3uxC^yL$Bd{_i9~a{`>XqW$ z0^U9a;J;0zJFrX)KYK*{=>6ofeet$FM(u^#Q`~2*5o-2Wka8B7722&oKJa| z%ky^OWBYiR%yHhtq;6s=D-Qq56L^wIjODWq)-JE3!?2wF!*m}uO*kN~Mvzaktfe{J z!fBD5JWd!iY)XEo^4k3~Ub5EspT>IQC=4uc4xEn(Y&?2A8R`Ar05gQ%i7r4<00`NG z6bl$z`G@wXhKIDp=YS?$W0P+(r4Z&XO2_g*0LoCni@$E#zHkb5aLHG;9x`~c?giCCg230?v=zo zpnF&H=u#v97-BFG33N;!3+zni6cTP?i1z6B9o-$D6p-i>rDWfa(>T!hlP(Tx^ojkk z=+pB0nw4Ul_5T9J{d-MOW*T&Ic380~>1028Aa$6}hF46?D|WaI5^GQTmPe?bog71x=-Wn-F@;(aOrIC^%v#-QlB#%2nZ79lX}q zSFC8uR~m%u!ehpDM*$2!g7yVSZX(gam7Nqq4}pe=Ya+lLE;+_k5;)>$VZ1aSX;8e7 z{3(sTa%7%R$zSxa({9?^aU|8VvVes8qz#TN;}Z`(FH#H9rXxsRs=7yPsY-r;6@fnM z9-xu_5N%AN{k0qk9g_7z=`DY?_jm!a!P_M;r-MXjOXHWsaoxMK!x3~VuLX3=-%62P zjG%B%hz^=ZD(--@b@8%?eTQRrQ3_qC&`wcJXQL^&Vo1K?;&Of5r6xut{)aUBL=Oa_ z|66>|EB+$wyf2F*I~J?5G3Q<`h+@wcS7Q*G%m zZKRgjywX@yS0HDH*=968=^rCxjHxpir5d%ZAdrmIZ_zkTgY8|2+JI7v!Qzi=Hsz2T z298z98LvXTRgQSF4IG?iBH*eaWywVY`Uvzoi-UqK91cf;!+(P6G=UCK6J3kS_ElXiLum&ueCDI%QsggeP&1kEC?OP{dxYdQTdd5~ZJHH^|;7)OT{ z-;~kQwgL->3W;#V;r)7J4g#o}#$t#Yc_{MH*Bd@_jjfYb|W1if5GB8hvROHH}DULV(#Iy?#wo{2KpYjzbCkVMPip z!KqGtfm=;2fxk10;A~Geb8Bs8kx%ip*=)Dgq!dbjdaHTswpGBT0lc(GXzBc4iTrSU zj=H1E4?K^vsgX2#{LFAteR^whAJbi#2Jj`4-zEE2Kfa#+z1KY-VIFn)0g432d?LvM z6CBi73stme|CDEqVokPxz5Z!)|GV9%PY?dNx4X7xis1DWZ*eQe^Mkdbds(zjUjm4> zT!$8O6zggNkz(VYKm_4hY=7QZ=Qyncotjs`uog6?Xs4z$>Iru77JU?o;Q-VUC729E zQV7k$=bQ9ipck#n5{MZ+*3#AfqI02j48Qb3Xn>)Fq^;} z^CyHpbUA@F8Mynnc3m?0myirQn$zQfhaQ4<(n0O(cY}U=ERz3JORf0zdoL$cARAl{w=w5AA4L!JSem7j3^9B-tqDaa+W}l3dpE z5kK(&K;o5E^eI64GPblJCE-j)sdf(@qlnKqhV%E|u^53+hV;svy%+mWeq1k}^@rd7 zfGs*mT-^R}_P);p_dtd}YCj%$X8RM_X{U_}c{l~5k=9*#@>2t_%vDG(BWc)mO%1=@ zsUa6LeQZWU;CUw$xtR(AYpNYckZhw;hl(YRT%*V^&dYCzjt$8l_`a;xpb~W1MY!7a zR)3;BHVq;REvjq0MA&^O037+aJ}$_1C&-88zCfyio7KG3?SvH2kkjCOM`v7TXi9*C zPLdjgf(wZos3s4Nv;x}Rzd@O)T6r+xi|Zc8Ajk6*zYZy#E@y=hOqd6;P@GB?&?&w4mGrX4ov zs*CZ7EGh70X*k((-It_@>9v@AHh;+e^iMHEK&qF_h3uVQ;Q2tjlc=yA1u(IecVTJk z;}!4l!it-0AoMkHeUi&e;X$+S;Hlte;2tI%_?$OySnR!g@qc!=5B7`u57fuWJc@`_ zZj(~O8;ldAn68I5Z{XW|-@SPL&-(?offvGlLM-@y-mbfxRvzW<*MBvG1IC-@0I=u5 zOODv1mglxC)xTs6D$~3A{<`%qg9-|imlODD_iQuui&D$l@NtJDNy!d?8Zw2((kl75+t zdvqz5skHeRZuA9q=RuXANt~Svh01x)5{F=HV12Ip7B}c1@<+HN&40MQAiMC+3L97( zZFw#iF;)-;3zkrP=QvkL>l^{B=7Jd1#DG!>(7#K+`#8?aNuwHa^GK(^J{8Te5MP;S z9O;fzw#HhKXE0y%5i~AJbag;T<^(JUp5u3iKX<>E7m7%zP;%{6=PMitm+Tjhf>C?@ zI`Wk`iE*;R?u*+l7-3#vyPA3F2;50IKqT&EhC=(a3BWr~Uam2Ta#R3xoyzi6Hmz>~ zEyoPVKs;8l`T51Gm)pBycRvC}%=MCu*FbhQo+$cjFb&ZmdI%SAnJ3l-4Wl4_gwXr` zWv{!2UoW2R@9pBo8Y@y1S#1e8*(cB=xF-r}b6X>J5PkayQL}&=O%1nb?^l?oSf|+&LXtR8F^u($ov_?1V z+wJeHH*^vC^XU#QI+9<3gMF~w1LDeT2)dOWofuaHtS8U^vHff()`wU4V*lXLt0zx) z@P6Mv*dRG(Bo&c%jX({06r+HRv>YNPq*EA>`SonnQ0e2jVfL^(p=(OS(e*;f7zNjh zJ#iv$a|LY%m2VQ?9AD6{3~cBmPXoX#GCoBSBaaOSnb3Zwr9rvgHP{s8v#76+3Rj$a zgV|qi1kx~CE5;WTrj1Fl@vkaqFXf0Qn#W$bm^$^@AsiDoixW+Glu8>rr)&#D!rhdN zPTN@{d_&tJD|KqFIjk*PlUFKJ*zV(~qtOHW=fuYZb4W(*p(3$aGtq{5U9nCf^lxR` zC#Akq{ws6!XjF^C_vf5w(bQK`lk#h*eNRE zZBbY`!27gmU!8r5B9T})%}H6rCQslc1URW}PJg4o0>&x(WbC!7qc92P`cLLIF=QO0 zD@Zw}(3txuLCfgk5nNHIJhKZWJiuq4$|30xGxIM1jeM!K9zX|rIjZ4J626G1D5 zZrFMXiZiMJWoP&Cljm^wisCa3`N(8JR(rrB@Zc@I$C5N91qoh{a86O+s==qPc36Ez z?If#5ASBO(VyY+(kiL@Sx=3J_qF>aLyLQ#DR-!9%MXD9S4$$_q_osSbJ{YZ+S3|K` z(4Q@Ji@KJCW32ajDg@OKjVEW3sIH*BJYq0hkd;N*;cFaR7F`p$WHzYdPEzwlA~K^FQ;kka-_!ntTR^eJ>H zixL-wG-HiWx3L*{B-j)Od437cM^r@AovJKm99(OYbde{lDOR_>SUDtt=Poq5x{f^RbgO>zh9-8 zrbYWDmM|k>ByndiO|`(r$l1KNmK6v#_5lz~UPIg<>~gXb>s+4A$;qy>4YxacW|c~# z*)HPqpa9wE7!@ss3Sv9~evwSmDaKrjtf>i+LVTh%?oxj1Yf>MvwrXB!CvFOvdBEL| z2$hyn*lhsMEzc;jWlQGCsz5XFK%&rq0{WTl=B2<+-H9%f^^W@M!7>_5?$ zni(OQ&#Vn@&DLExp|XcHAu^n3#tE`9d|8 z_%@Qjk}TY4D_|rWgWhCHPtrk2O#_eEYZsd((P4>RaU#~lCKxGiAwd-Invj98Rz%DR zUqj!a&dM>8rrrj#q&&paf$X2zutdJlz7IX(MX_867;IFgqi5Z7vo)F71E$GTnN8Qv zdE3ul9PB-O2{ze%nuMu1Ee2TD)6kWyB*vHS&qZ{;W_cyDZpE5?kR!|z?)v75i+0|k z@u?XrdZH-u+_IRIq!AM??fKo}^(KPy)A;o74U~J#X8z9L0BES;0lex#+FsAL4f{*mLKKmlic z(TGqHWd2YD{U^hl52lU~7>uvUrd~9rdYoc77FtOkhkz}7F zxdi`FSq(}<^jTa8p_0^Dhiv^5wdl$>8F87FOA0f`_1cUw6_GS}}Jvtgod3&rZxOnYk zF7dXY@J!@%AS~WnJ*(PAP!gs%D7a9WjK^g*&7zV6 zq(vC@SwwGJw_ls^S;R+o1sK-I4I~lS?V!LD0K3+am&VI=ki4@CKyBleDIBd}5CSo7 zGAsepwbIap4{n28A?a|2&-VwT_lUabedit6H}z*{pTpZ-!Ahw%?;ZXB&%8Q6_=AGsZT^Rp#b;{t8f`uDfe?y5XW6eiI5n8rbI_Y7rG*{ zs1D5+FN#m5XVJY@l+hpfz0!i@i#H8CE(mF#39l8q5~BvPU%>Y!vQN}gIH>m zH>zoyAaBTt(sB3QVd9a@bYwnx|R?DfbwSQPPE95{{B!U=9yiN4w8sN>Dn{L}UAyscsBM zgVEWj*ub%i_doqRn$t-ueni0~YDpVVX_};R>()AMWKdp3a0NLQN%M{{8~w8+C~;u^ zfW#ml9MjD0zk+^oXb}%6t7q41YG2b~`fbyQ0zwek-pq8OSZWTc^ExqCYs^+aG0P3} z{@QdXM#XCo*W3u#Q@=)k{SGLoa$O8K1SkHqd0FhqC&gZG+)_Wtw$#AOx$I+VEMZWI zL_m?30!6yLo353rca+oBVt+mrpKS`$rKA*6jYDHbw(;!$}71s52p}w!M z=eTGa|IV-acYaml_1ug$=$Wz_kN42Nn@q1x`Xj`PVsJOUa{;tBEA}@=8(41z^C=25 zem5TC=l0$!Y+S$SU&AoU%V+X)&jTI!sC#bZBMytavxQgJU)*yb_{kS{w){mXbMLOd zy1sW8W%l;(_zV2Riyx0i*LUu15*7X|@EdP`iF($1!1F3}O+r@rIGfMT8U4hwDfszF z?_@5%E*kG9TbrAo`$u10VO%3$83~lo)e+`Lcwz00zE4enqb4%1ZpI|r0F5IJvT+Kr z9lZ6Cf+tel@$8U81!$76?ODT6=bZ`GNJ-2HOW*lscWf2)^*?+FTmQY`@GdJ4>V*%I zgcfM|jnNYKykerQO<9E1^r^=76Py{q61GAVi^g*{a2u%GxC#-#nc1;)17PL|-UmKG zRx&_Jp}Vck+hn<9G03q_)`R>N{dl=ituiX%Ml}TJ!NAHVy85GChdK!IyFD3qk3wq?Lc>4?CFN@X4W2iA)4WVse5GC?6U|};g*Lk#2MpBa47g@c zRbYk6T+zk<;g7du-3TQD!w9hzl0@DfVA+Ko_KLDP{67LqKR5`3$U9m)1E3o&>=Qar zE_8#NQA*yrIA)9j4+!CA0dBkm<}mHYZ-Qa)e4Q#1obbJ+^AE7#t=?c7aH?w`sVw_s zSQC1JC9asyF|Wk1%Uj;(mB2Z_V|+j5JtB+BO*<3Zy{ii|@TwgSW z%bSNl(Iddd+7+@Bd7GkEl$HK>UDlfawauQ)E^<8|f_z>E1cw0jhh(TlwL6*Dkba%p z&Sn3EV^#}%a&n4K)I{~<*Xe0fV#x^vkrE-~WeIltQf@W*GT=2d5uf?jzmZuzyMX zSMh|u?@w^OHtx<8ha7Gu%gwvk>RG=#9!*B4)1q1x4q<-T)o5ZZ#RlJFxoh+u=RE&8 zPno)yba8P-<(#R>u!qD8oIZvBmun=h$TVPbmz~^p_^~~SU4Z4l@p?M(p&cQqk%2Po z;hY?1j_BD+PQV1P=xuPCTY{UO2bG9nM4-zVMcTeYcZ|r_DuJnfoC)BFnjXF2lw&q1 zvl$V}yDreUPROvRuBz#o02~d-gb_3|@E9*|z(*qc9!|INVVG6HzK8Ov4%MsoZ4|FYbs?avmNIrcUJa6|W}x3^2%*{t9k`Gzlr(nW){)27YM1b*e`6`bY8 ztJ!R#Dr$rNw*Xnf|CVweC-4a4JHZ%n;neugHE+h8chTX@f8?||TEYLFX>;2CA7sWo zBIeC0XC7IoEJlgz-0vie${aGy_{{;BH{l;=L-9asJ^{bI862x4E1?n5Kl>bfmalMU z^KY9n@2*iQx2aU*vWv%Kqzb>s#eZ}$?zU%W8~s3SQ(6D^Fc8B5VUv?7OuX~qus6P0 zN<&B0?+z`-q03PwjZvj{v<~~&p5H2tvG)Y0)$`^q9)7xlB0X%}e^9Kx9={&0t^iQm zu&|W;bmM@lcwOs}jD1@|X133)c;UX$ws;Jkw_ZG!^Gyra)PHMlZdz6w0#TCGWt{8A zMs|3a-jt5TO*eeYC@*QmU~fz$mq$sn!g^943mWWM3;}yj3@o*=#J?@`XiuY?&Bo8S zCcj`co*`~%YyYpjMe3YG32|ag-DLQgzZ5rpcXidbozT0(69b>*Y<)Wz9d~Rq0*;B* z$9-(ug#X%f0&b@_7ZEA&jej!|_9d@J;@@#Y6T4os0E#i&T3=P5&w+CL7B}HZ!c@O# zaX$xE(+EmM9L0edu+NY~yZRw}+q530_Xi2WSg}Yj{uq!|BIGc+ta}qivSncFYO)GE zmulpIHuhmZ#0b6XZDYNel=~kc9$Ig6xC8x?AI>o$KSaofVQ_$txV5Cs<^GQZ2Td%C z88@Qsu5Lfq-84FqSR@C7<+S~Hn7EPa$?&EiBeR*LRQ0Mxw}I*MYBf|TX{oS2CFGejtT(+HQyl1-pt7b(wbm6fIY7?8>P{`9=rKnvx}p8cz1 zUIomXv~-at2j{t|aZhP2TW17iKt`k@sFi{+L*H^zLwNwj#zP|dhH`v2RK%l{bi4y#zEWV0z zdFUIXt%P<=Mmc1@e$W+O$UAEJa86%c%0igcb5gU1!a>paMmW}4+t3B(_Wnu=Qp){l zi}s$q5Xms{2Ri6*xQ7#4dG)duVXZO~rLHYH=YCFuevaK%ugjXcAZiASSJ{77h44?Ju`JN17pj!?A`d z`Zv3*0s!~B=a4vel9qsXn{H{**uo!mVj|Ap!Wa=}cU#;*c~`B|sPMILh&|uI^S<4N zq+|j0A70|nXI)5t0-u6ljSBoYnIJ)a@Zlf54=~f*r`a3Rv#2lSUzAaaln9!-+_JG- zoev#$hy4PxcV4VL8lx5Rehz#`9oy|e9~yhVJMLdjS>XTZ&xop(`|jY`)4k4F?`h`) zwrR2Cx8wdvrPzT?19VJS=*jNqn`xmz=fi&gOsF9&$!}nGPkcy5KCn5+^?ZbbxP~X4 z@yXNv@whYY_vmal0t9MH?Yv%M?!4y&QIS#9;jP}p(*s>@5eiC@G4lp9a4E??l_ zm~M(bn3n8IuMS!|olfMZ`TNdF48zyzQYwo>-|0_5LI5_+Utw{e$FkXg}wIfW$&~%hA*YJ58E6MiI&Oy&M5|0OLzak0s)`&97f$; z1bb>*xv4AyY= z>A-eU6^$au^c{|ZCo_F!l=^YPR&P!hr+P>b=CBSHbjpk?h!SR7a;3Fjl}I(b+$m7V1iOB7h*T_5ilCsYYk zQmnw997t*aH6C$9=;UD^9L9If**@Tncrs;@xG9km? zi|+qFPxpIcxCir!(gaL~bj)v)W#Z8XZi)4R}lna@!0-CFs+-+NEM_9$8? z0kDHN(#~gXtD~hHIv|P!34?(SFeYF6dHlD{`toL?Zi^Y0oL|^{%`eMM-wV1o}puzA3_Vjc|Cs2V8xH*lvI1nXcy5 zkPa}X+UNDHAx&v+HSAjSse0NP(jc^&#rVrym(BZRBZ@3Ni0eteBl`}lRg=*@dr&j; zql2ct95jf|Id=Rg(Yo>rrX2tx6O9X;*N05fg`l#n{pRnSo=q{#99NlJ#{GBFZw$lbz0BvgI2#@U8}S5{V^^N38)n3_Z6XFh# zdy$?%UP?NNUFK{tRqC-SD=fbxP`?y_c-s&mR?d9EB;MTr%ig;HS9YA|fkzbe8q2mE zE4CcJEI!x?&p0`a&2t_a)<-g_|vn4(Q@w2tESL(a-s zI;AACEmyP~WmnG171xGY)321A)mB^^WmnmFD~coAO5!A}l*>`d+0uUB|LD_w?zwo7 zR7v(4w^TH#J&G$Qy|T z1z_ZzDrFAuX+fHnf`i&y(TZmZieB&8;$6F;3CV6J@ zU5q#)-j1UI&rLw<<RPJK;mU3afz~l$?;+e2@f-je17imd)W2hEu8p)dXUw25YC_st|8g$uZigF0AE$qR8LY;ppe%)@vj9w#{7B{A~RX!x%VOsL70pzWe^b-`l z-3E`I!I|lLMC;16>LL`fh*g<1kdMEsM;>#L1e%8RTvRYx9fW8}ih()MqJR%0vCq#GaLG29^K59?+W{{{40dHgg)KO!eumj6gr3El zET+p9)^^i+aM%P9#^$PXgi)Z}P<;g>4lRCLtoodK-IPI+@RO9@m?*E%C8p?ucT}h# z52aD+V~tU@&pj$2x0+s1)>%@;n6_!J0dRmfBFZpy#K3l19NNJP19ECCv^Fl4I&>-h zRZ1#%^U2bLQ-I3=gp0?hyX)i)C`P&x&t!rxE|C^IIa13|0fmwdV3|ggamS)=LnEv+ zACjqt_GCb#1yy*691 zD@Rxn_G05c7X+{bHf(!T6`&zj&0d~gTWFE33(g(SMORf!A2C>*D~M39daT03Rf1P@Cukefu!n~NK>c4M^@n|yG5zC@#v=5g4S3`t(fRAY2! z8DhS0)~Kiudkr;XDFIfAd}Y_|1G#MpfL7zt@ybbKD+!U98&#pvbNZP-kzoOtP*SO^ z0UI2zF%svM!u_$G+VovM5TB!$9)f3~Y0!8nsw$eu~qC<|OF4aikowb$_mA$GBgQvVO$3HzH?gq_F zaTnB^fUyhhp%&K`+vD}v(w5MeBFLv2uc_EVWBt-Thlp$bGFQXxrxE6rYYuz!4Oj(f zHBV3|>_8W}==1C>#^ zKd4P^a&+$h#4s6LRZJ&kih_-;*x#uUsO_27^muF!r%2>AsuDpHP83(fsHp}x2*wnZ z#&wRb)f>%f_!M*X61)<@d;?v5UDc7Oz(gAX)Ix=O=l~DNs2*N$o z4^PPUse})*0^?VpbGf2tvR8@9Q`+egcXKv;!@WvW(W$83<2G2RB{af8o^;qjtY>RUXFiJUmEymBQb`F$(b4Utu#@t7V zuO}~3h9p}FVGs+0PML0B<5qoTh+0lG;m|)P?q=R@XCPq(kD^}#jxbwWz+qeV!hAWf zHexz{HFyAl+PU{^5aDr;<;c)H9V|q455g=eB9iu(wZ<3O!Z%%-uCB1D$4b}`zEARo zq6mRTc|xX0xJtY)_3$2};n{idi`m|a)UTu&b75IiOH*x%w*}dyWqPe2Cz-JmpZqp$ zOX;A)-qAE9QWB7_yNbgVyrCG1$pHk1{HiZ+0q9%w67xy5-I_NLhCNyA3|bH!738et z%Be?`Jh(j;gQon0yhyy`7wcllhf(lPBq7ZkE&gvstII(Mq7$zY8ko>ZB9Re9Cxj-$ zC52Q4hzoBRdC&jIG&aj zESCo=3#k>AqRP-v%`1x@(Fes^&H-cMiXTym?w}^skY#8k<_QpEU`K@|9Nj1!OsIQ^ zQmNf_0gF0iB9+p6NBJ}l>4T13Z0SrP;?4Et9GTrx5jP;GOGN2X4@W*OSKwC^hrjxo zPeOqNTU`MIZYO9JcLwtBi4$0AIKbi(P!8y8l=Ca4O${Anbt=Kbv^Qeg&M9DI>Fti? z5Zq{aE|IVtF%CjdqN-H50%%`1qjJ*YxK$iz$XMKQ!kGtKcFbhHGIEPf#iGdN%9way zX_82@^4Pse0mW$5s=AQ@qr?Z&tYR(gRgj2{m6%2l&>ciH#=Y+NRiL8r@Fn9Kmxi(A zDvb_i&KjZ3bg$=6gkt1~0Qn`K>ng$QmdwBeKr!SBYdktU<~d)`aZ*)bPt8J;M$?A7 zH&mY?2I=Qn*j5_c*R(KkE?T z4$02`3|DhyN?kDzrNrkG?5iky9!OZ4wM!4otOJH-6i+GD-7YeV#dUhkB1Nfn3l+T=dhz&5>w5i>8g zviM0_s?&9aYuAY!PGCeCLgi7MQvw9+hHU0$hwNdp!9T(0S$xwtZXcRyNxAS^%PU?gy%_CSN?2{4z zP?~7@$T<`)02Ml6R`=QR9=j zx;=NK#MeyXuv@LMDZS&k;mkfH%u!U|9YyplJ!vA(RWgb!4ABK(GqwzqwD=p3Glt7>Z_hsMgBU5E7L0%~0J>ve z_b=smT%0OHjL7E%S7FCD0rNcSN^5x%>wS3y(-iPo+FCx(#IAzlBrJbG{A4I{&-WPZ zzZQmpT+E1y;$nAzcmJ~AP~?>qh9l;U+KQVa5DMeb=op@{9c(ntM+@s#*!5Ujt%V^d zrAXaIkaLG-(yz5~`HGK;R`4}B9^f409xgq&;IL+E&AM;S+gozNgc^Y0JFiw7 zm4UZx?LbV#*aRgqv53w{2nNUVm#$tPAGs8cGvQV#J)vtgGElAZhRoWkpQI+S%(b!n@fb4v<*O1W?8C)LuOZ$C^oSzX-{;Su^a$7n?6|I^YrMp_YEHR zYt`4aTXBg=k5DCbB>7G|3Vl|1OtsSo3`1_}>`Icjoiab!B`L5>yDow*kw!w*vu=9W8qSL4YpHHN9La%iuk{$!nm0?Ba6Irv%xE7XZn{0ca#dGV8yEu_g z`g18cz6C&wy3+~QuXW0dkvaVN4Kd|b_6Gze7!0N>+=By4s!~;}B1Jl;Cdx)^EluE9 zSwION8qZ&pOscsl4<~Di1EY{2PT#^e@vsuITl`SiI+mCRbq-<|1RTBk*IGFIII(b+ z)B|bZd+y&kImF#ky>N%@{hnvH8|}xiY0AB0)L$y9W(Z9m!Irq4l=6gY2#{(ryIGnRI-%l*?kRPM%a0PeB`)vF?gcidQg*B&;i>} z0-zc;;~Cl&hB07Ypss*SRwNiWj2%Zqp8+X}4i3^QHap3SS)cMeT}sObz>HN$?NN z9lVS!;~G3v#)Uw;oF=Nt^?JJtsE|3R{hgm-!f#`fi*j3|eSy+dV+qO2j60>wK(8-| z21_IQm$#cZ0Jz`7Dv77`kw;}^lrI$r%4fhvrNf&}?W{?XMK)ENbv`-0; z56zcpEeo8Ux#@_NUrl6%uR>1}#z+lZuAd;djTpb~*Btq1S>@n>=y=SXOZi+7hxg6? zr(FyvN0VT!1bhx52m~wVUJ^k;8Ynw^OC^un?L-{N&4Q`Pww!F2_e*I^P9_#O{y^&m z_&#hwey#b z?cj8s0NQ-pBRa04U^&00jGkW(wP%9}tb+k8r=La1&?a~y8ihu?p!>SWNC!Y5owW*N z>eS9%aT-XYU9Yx{S_qj^wblou(lb=wtqjjx$9+Mxy&-clLL3oKafsmx#RI{Lx8B`4 z^Hjh9tu&;LtCkVsC=q8*o;=rYoj5!;`*7a0>uC_kL|WmzJQo)%W(TjG}r5p|qf2e&#Pi^~W= z6ZmXxffwrjadEsWzn}}bI=|xbacry%No-L5jS8vaW+e+npSwSe;IQ7;qhUt`m;KncyIqfH@%SF&Ttc_>hyH(8T4a zuty^fJ$yDzs!g0?xY99zrPIU@D)wI71gbkEw1jl%!NPGaPTK?x&&3nur>b|Wt+wXz z_-=RWo$C)5PX_pbaVlqx^(`gB>g%iqH6vp@E!RiYc%5z@)KgG-*siAcMCHftmYrAY zQ_QJ#s3*VSO2jt(9&Xk1Yy#%5&P{w6DrZ6BNG9M9mU%g~@*>1$^LrPoc?C;?9jrr-s*D9dQoWye|w+V_sOkD zLeej5!6luwwtE$kAg}D=M!n;O{T6MI6lx)7CWA5AJb}y&7xFTMbig*m-h1n1+ z4fzF@j*c!f$t{+^s*3|@3^A#&1QK~ZgWijKUiPrvUOjPMZ+Rfh2-AyDR=7?BsyU)JI_e$}Y+ez} z>#5-O2%aCm-fnMQ7#L{bEceaDR{v&gx!GtnR@+$6*c@2l?O{}U1~yy5ItI9i*u#N} zV6f-dwCn;*9UL8QHjbLq?o}MkwG~sZ)ak1NrmbfYU+Ssc?tP#y(Zi43?j^Ih-HWhC z5Hk79Mkb^!w(~8=E?TWyr=HUFedQaj;CEE-A3AIHFRavXGYAUKU-w7JPTT{6$Je6z z61lZA`6v(oV59W;gmUei>UET$1NZEFuv30=MVOw+b*h(fNh(ODpo~oncT@-&&=T=% z#MP;nxuS#PgrgW48~1Vyd6arlSe4F1uhCfM!VLF#3tFi$Pk26@bDaw7JwPCVuC;@4 zcYHOyT=FK)l+uDU#u_aX!H$b@J{h;b?r@y~)09HuTig^WyIu3CBxx0=qgCOo`HV=4 z@X-wKSfVZx5_lO@^%z~|Aoa?Vr`V$NUryX*QXjWFq~u+`p~IoH4phO^>*Yq1Qzv^< zW^}Po8U}#Y^UEF8_=lYR1QG~CgO**8C*)lHaE0aNbEi)0QQvu#6=sBnNS9I?637xZ zB8k>Wj)Ff0CRs|TI_N<{Z;rkHfj%b#fk4O|^g^E^45pw(8<>byDZN0n`pX1J*pQK&H{MOrkwUj_m}YT1UKI}?f15J;$e^BqcwOG^ic5?T)7<4V{Q zrY%fj3LsG@7~k$EM#4hfSbCvTGM}*fZ_k0obkI%|_gNjL7sy-eMjEw$KZ?~Kx+7&n zww8FO7cc1ELqY;n3h0G9nBqBnaa>LYtYGqxhV!U96Ik<`XJ#Pl9lL!ye2JwV#E135 zeu%i6jXD|6R;wqTLZdr4s*! z3NI|pODIwYK#=&U4sCXxWDG!1juAQs+NaArIhOKr6csZb`53f-+AWWLb+_!QgrXdrsG?1f(-Y;D z#0d}-($W?t)(=lgAX}rk8!=PJuGkz5I@^*PmZg+p9EwW(=fa2@nYB{`sCURV{s2q4 zXLq*BLohD`JvX=77YE|Fc!=UdTrZ!!IFP&}VS*8QS#<#+IPjO(>{*lj@uxnbKbxM3O9a6%T*3<2qGKU`X+a10M>im103X6 zl9%9io!ECoQYz?^d_^t6rH*TXl$pA(*rTO#euG{~>Lag=opq<8DeFXz{4QZ>CsGGl zPJSSj_nEAmIfRu*c^Baa0grWQdc4`}o+vxR~3xyAS zGPq>}ZX}TMo8664npFi?(-9%IlD;^L$yEv<#g!V@LVAQ*;D|b}vKzZO+A=m1u@j!@ zxYe&T0}MsXfL60zB79ECib*p-l2FloCr&^;uWaIy#MRx(F0PfT2nE7;%6$#6Kdh{1 ze0#C=3`VDg^}I?0*ReKMIqN`aC~&tbOVEq&ifc7awFifTX}MBs_eUfH6j&}Vt+YUc zH*iFEIO3wR#oI7ihyVjTUTNWC&ZvLNF0G!tK$EH~#6alqRUnc|SJ1{o|4fjNV&ggg#uBZ0iEdM+$U^>y-=hRG1%Ct}t^Yh9o5= zb_q-KCMyfWzcD?NK}tI+z;qfE#*S;uP858==u>{`BtQ|l_jy!#T3e5`0y=S^xV)@- zoZUhftU%m_6R1yRViEJ{l3%Nvs8uaNRU?}a84kPi51^16kyMmTicV;zYUX;(qbX=% z+nySSxS1fyFVaXF(|g>Ss3Zt4f{o1T3Y+8#FL{QoaGHZP*~hO?g9-;oBU_vT41JUp z#~pVo2K}{-#?sOH7HPeFIJn;PQzQj?L`g34u?_6S*H=>EDcg!J}P-+ZrcQVIo{=Zalm7d&SmX5 zao7Tzrc^mw#vlQtk3&8JGM&sne4!2FR^PH%AKfm9D=Tq4O*pC;Gpc7~7toR0ReAn8 zHcp50;rTVZ*he4;92j}@GruC(sHseZjL~7s#gEvCZe*=}OIsx8H$Nr9X910Bt2Tib za9&UF&4F*Im4GwygQd39P@7stE?v6ad$e^MdxXkrYKYF#5}8g-F;2DZ;^&@`T8me{ zW1ZNj!s~Jabk7)6FDwG<64ybmfrcMjeU3=IXU@|5pt09_XHLEd7%1@@FtO5uJFX?R z9E)t1L4tRLmFQHYT0(R>=Xh^_f470e5!Jm;12C5P{$-HgBdNUW6iz0v)WqqMWC|wS z&e-1L%5u^PA!aWlAK3dQTCN>=SZxW#?2HA^q@3A-FtUOikmPs|mG7S8ww+N)xilDZ zB&?}&JFimmIHLD>sw4YNy*>5=1x{j+YFBpK6w2L6iAz5e56Mr3^q%IdyK@Q#7P=IJ z*PhXc)3CVsDL-6FK`wl-U#(jvJU>PH=%3x3%f>tZ5J9@yPI!vP0X~+sM4!Y3$_Zfw zcyX)p>axRHUkt<;akq7(_eU8iqDXHf%a4~$l-KT8EqAtIsJ)T5fiGR!AXhFQu>LYm zGCVOmx4k90zglOa2y%k`=zEbsiVCx^MJtyIIj>%1(V_u#V1&zH5IUD-TC(sk&*k1- z(O}})U$~9eLD6dgLk(G~UvL8k z1|=NpVDvFj?V~LQCO9Y8GQZ9vuEnVe2^`||jSOwnx)048_7{&i_OSP1kTHL6-@Z6t zFPOhE(rBw9V_&eHxd1Hm+pow>2!xsw=?QZ=T)sl_GKw#+%~+L9YQa*uBiCB07v`39 zu|L1dA!i&aW==s57eW=i7Akjb(H9A^g__0#GZqETsRh8Y9u>#8E~N!CLa{g_x#6CT zAW@oaU$FuXxUd`mD`W{uO}+6|KzfG6u_G1Z#wGUwNjuBp(c(6iI=E-NPwN5H2=usV zU9LOgpz!1@_;~r{o`f1PbN6$tJP2`iEn*_p+RcuZa$^yj#&yO@OI1HATo1Iubp)BvQj$&)D;t?9xM(+3QLww^ z!c8ms&3Ga>x5zQLC5^WP`WRufEWD13$-V-#6fO{o*9f8Zv92}xpLRhXsmJ>pxgxwhDYDX0p4EyON#Xb$&qO z%`(D)H12{6nPZ)_GuKdSx4w)uRd1bN1ns>jnWoKlyH?tV1Ky=dnrsY@Mu>43L%dhT zX!Z%5rlb*fmthK6XfFtlt<}rg1N+pSYI^l5tJdY#lDk)v<=C?CWe_gEi*@gMOl0Za zknY`v<622q!!5^Qu-SmU?n}Om3x%t&U)I;=#wSN0%2u(A1{)nI8 zT!LH*07(XT@`C35Z*1*Sqvr2gXL+wh%3Po|QG0Bpp0cOJLef-s=^O-!CyddcM62UX zbw||q#)ycCp^S9y7o>T$?BN!bAnR(TV`&kfZ*USGs;|J(JX2f1B7$flgXNXGE!8CH2ULxEqpbqQi$zNN{k^c__eyBYeW{06h(hH z_dFx^8lM|#T@Js7xoyF2;N63B+-aRUxv;!pHEV^&MTa-60A>eXibx}1B8q`BB?2Ft zk;@v>ngXL-fe_pgp9mhFw)eQ*rv_BUN(`(+xfDzQ#8oFJIvS#}9QbX?8i=4*6486S zQCuTIuk=cgM2096H#SQVPIM)2<^J74u~6vzbF z-OVMeyd;n$`K^eJ3S(xu%bDxIj@emWhnKSSXh*er74uy7mG$7A zGh7SqwF{kb-7okA}YCmV>x}1s-gO&Z!?Q0(D;;Dka26O0e^^ zjbq6$+b4{Jc>h={x*$FX8QjZrXd2`6j`ws~q&eUX<>axtWEsmNX-v_RAzJKFTXeq0 z^(yZxXV=3ThpmuueklGJOPD}JM3%70TQ9U1w9VguvhH{hUYGTq?WE9QE6DWB8H^mfvt!M3v^C(P#M{pr6TX}^yJ9UJhvt0l3&@EoL!R0 zaRWzi77bj#^MkSd7{So;cxH)OY~z!oyzk;(<~lZdA7FW)2~}KN&fWTi;ev{!24e$A zu16W-j@af8Gy-GA^@hEYBwANV2~fa%q`301pDo>52@5)ot4{6c0B09}C2Fs|k+Ypr zu5{4}2s*ha6mg}n995<(jn>=ET@mlpkaBqwd534`K^&Xy?Je4`3JHA73q;0EHLIk~ zU@#7gaEc6u8PkUgFj&z;7x6=q3WnGa!6vo60Z%Msl64?izT(1G=?6?9E5z&^C4|#V z+d1k5S-`%Kkf3%ET2q^t=@|RdB-<(L>>Izvc_# zRR^}l^>yHB{4v4M_yd;pQ{#YCi8h5RM%m+nnaH`a4aL`*zX`xZR!Cpu9AC`S2s~vk zpTpv^7siJsN7so#mbKRM)|PzXKRO!PLetOc@YQPfMa}IGO~s~h z;?`_R?~s4(IqjOz)Q0Lo=4P2+E-X~X9(1<6@U`z0zsjeZ~i zL$UGYHQ2?KK0Ty$fRi=4b25eWK*swuH3@RChB&FY>W7DzrzMj{PIr>i511!Qi4Ksu z-2)dj+dF2}b!@<4enEWDX5DVXF2^JzG@u)y=r$lq-BpHk8UYDcE`#@zI@~|dSpmm? zFQ@?_U{H~x8P!-*gj`&ypA5?qpHmW{2iueOMliG{Mi(A{J|j=WRyE1ty;3o_C@Xh_ z$@RYFb*1km>s?cgnPFRW)_Oxns427YPqB6BPAaQL3$?)vv^`>7VL120Cxv9#HMYX~ zI;$<>k4rXToWk}a#YX!3rDg8nQ5p{nAt|oGCBWd4VOl7~)~?B8`EMq-J*(Ffy@o4$ z@bXIUAOn!T7+f?y?WlaD&jTVAER#o>cK*itHFb>|Fd% zv)0&@8h2WlOpX33E?R~aLadD($Ah28WVpiIauBDLYP=NDy&dN>^->M@bol8TWu#u1 zow;urv`pOm?G%?wJ=g&C+K+T9P<5Z>wukJ_@($$&^>qeN)F{WERVqEsgA90f$Q0O$ zoDgHgOtV^4*_Jf5T*T*2JTW}4J8ow%$vV8pA#E;V7e+tY#lt1!?1!%zzx%+Iqun$Z zzI>;qDkC-(6!uPzIyOY_4Tre@NfAQRd|yxS#F(=)xnyglr7ya^L+YU6N=nvovxG?( zyl6V+!w!j~!)z3+^NmtRd6(5wb;hy?-!e>D=H^t6ZgNh5$wx^?4~{c7vr@-@MZ9uo z83OJFEHs5{586&AIz{QIOE;84ti0MQdAdnMC@^B=EI#7mc#WABWUxb=sesICa3ig? zJ_iF;2~#VuW2-wzRe&AMm=3(;tT6IKq))e(I-EnD4sb&VQRf0P?*wC%<;`Zj#Kbx@n!=915qdhI ziq?1?iy5`{@!s3@2l~3za10mTVF*O@X0OzQ9Lngt<*uVQa?xGfCG)vSYV1f9nJE&G zm-FuQ{Dr+3_#Uz+72j^!NUbqvs=R49=2z#owiMq{0?!RRK3gjIdtrR87-pujB(bN(EQV$s_WgW_3B+c zQUTHD|{bM5VwTB8EpiRo6b0@|#J zw~VAjI$sFSkDV&qy~eC)f8^qUf*P&R4ln2~M2N=S7Czz(j@rrsm+Una01}q!u}GS) z)>l{THMo56TjU0TRSqr8-Gr5^^$auJ@IwztaDPuoCTAR2gPuFGyzDarlo*Svl5qIKahxrFL_7<`?uw5tuj+MBK72j}^cJn2qeuI3RS_ED!|E3oc{)+# zS+c_~Dp)u+NRXG^{pKc5;ianB#M+lzEI4X_ozdng#cNbH-FZwguWYpMUVsRRMp(Ny z!>mCl#?z!`qP?17dA%Lw1z0ouV`pc)6r~YB!Z0K@-95{KrBD+KXdp*t88>Lqt;7f4 z>+xYtzK0AxrX|x`GT8T!UX2Yv!cKA^>GeA>K9Gawj(DTYwuh`eGLbcHIQxk4XZz1# z?#T@B)KZt21A*Z9PsvtR?4E$=49ZC97Guhj?n}2Sl_LisU1!kXgODz!dzGZ?47%`V zr0ba~geF|J1%v#bkuCy36i-snKOTMu1 zm2weD(srRUH>I5Yhzs4i;OnYsQ)*J_gB}bWoF{%RZX82POXMPW1UGP9oJ%r?5{t0) zw7LSHHt@6jq&JaQ_7;mM(P)b964BZ+BqEi3G-+@T7SJAm)P+K36l**_x) zIfeMsLJ-1qhB^nr3n^h|nb)MqiKPRW5WGCF6-Tb%6tUvWKHDGiEO?1wmUo+vKEXXh z$;|#LQWS699^@zV`Tw^B$nE_?iZMAHX!35F6P8$Es>#3cJvZ=6N?JK(Z!nhDs%x}# z)*-Y=Md)dRe3A#7oxWN+V&IkrL9RJ3A$@E4%QL zFRyRaFDNSiyjfD(W?7`JGqg0hTji|%b+5b-ztbhYwp=eSXlc{>Tqx-7;+@Wl!wI=L z>}5fL1C^h{#z9g~rh-pr7*u7bG}h^wlCoei-YG}pPg=xnm!dArxLg9N!y0l~?J|~8 zgzs7j5|GQ{KViJOozm`#THbCpLA?5o^%rEmba$CBRC899=FnK|qDUolg~nDn#5hJx zMg*)RruiSRQysRX2g^WB#n32CAqmcegzK>-6a3V&_Da^`P={n38Cc02@dca1$GA0d zRRtWr+g#j&l%L0t%q-7sg0}ZU$-P}T*z!~MUSMZ>x`2E zHnusV%i78>l7#cd{|X@CVTA@?nO>y@0k%;(omHVi?=_>PHaORX`iiw#k8s#DF3;6Q zd5xJaipiGPvGbSj6&5YhS>Cm(G-%U4vRke5jLtg|}g)z6HSuR6EG z*F~Dg`x!~$(NJDcxZC9B64bG$ct4H|nXc1y&JA#QUaq&t^cW6X*#s2Jjm>@_8airO zPcUyk6*#m_{XWECPBPymG4to=eFv*LdfPn@NIB0A?{CR*zNjkL-w%{+MUN-%1fNhs zR}H!mLzVgu4y4=cxt#OY550Q*1l%!qs>|Er&~Y2Pz>9ElEZ!3^yqytxDz0E9GMttCLbnUJSZf7( z)+dQV7TpV%urEy@hp|NTPL-}gxlpufEBUGK<`{rwU*3G$bM)ldn`I9mjPL9HO?V?W zHv3KhgCs!h+^|Hf74)W1VPa2r8>-?egV@pWyz&%F+up~#PrM4%N+fQh^WF=} ztNlQLfI)(uoa`d9>a51^`pO2t33_?#h|dm`f^_;?qce(^sZxbv6%zvz=pAV&kX&+W zvHNAxRLH}QN}(LJ=kr(6SNf1zpBT6Sp(x{Q;XD{@7HqZq_z(Tv)^tM!UlWJ4v&V6d z0V?Xl`o5$x{|CCaM!uKK7x_hDId*XiXO>DToYsoO7uH(aOMN{gFx+c`P+U&V!7qh= zD92HLoJ)}yi_~bqG3;3F))RHx$8_a-_P{{pz6Tz>K0kK;!3XX`9>zVVA-qbA9Qq;> zymnYF(%{=NEHAX(0HPu8`TVnPAJ221Jy-k|Wu=c?iCj5z<>nQhS~Ui$*Vk@Ziyy39 zsbW({CI0H}7?lSHP?X>#_BYVr$c)FK7d7fO$WFUViGp-37$w*WgmNzyI9Fg42Og~8 zUsQ&F9~@v`amxgPxbDZqM+I)Dh!0sF9C)k(Xuzr&mwsI9;uOIt4voo~wi4yY z2lV*>iBULl#cfI9z`^VaS;!o1Y$UEW>8mr-Z0zg`h^cL~<-B!Hj8;GR;l>ISg#^WW z0hcfcodu<>F!UGj1Cq6^MV_@Cexu(MlR5y-%CI1E!~ ze#1i~T6Pf3z-Iy;66yGP1snu-HWK2d)haDRYf+F?J`WiUMV!vd$9hVmUDdmD#z*GF zUrs6^VtC^TW>@8MYSf>1MN8oUbI8kye}LRVu`CBb=B=$%a#oaq4Sy1cQyf-_mid43JrcoJ5B z;ICu5Ouq1<>hnCUW|t~gu8&X5FU-tNk6a&~pIaCipH-^sGDSo^BYLY?kZT%As?M3d zrz~T;nT;gSpwdsF5!e0!SP_j=aH&j&O%<#K?f1&UHj6w*bc1D5xRQwQ)hBtxcu;Q6 zr3yp9)spwr8+Yr`81%Siw6$~s6Q1;_=fEQ{H+_9}cywWQdU`$vy}|%Q%;}uKM66+D zF#-oUA=+TmnFw;D_&q8Jx9DZuYA(|_rhGKkZE^w~gPpN63ipt}WLoTUg(|YVj`<6B zp~Ys6n;8H{*JQ8~CE^0JLl%4Q(Fcz9_4Pk=?C7xp$Yf|k$fNeRye2Dwf}Gc%*Hw9_ za!R>6iYTqC7vhXw_J_SiI zFD0)tm(@i69}-9SN#5}aLGIMqN6wx<^YGb6@>B$?>UN!4Ms6j+-XSU+uxb_D>h4kl9p}B-*S|W@G2$Kr9zaZ zHhSrBV*4E!z|okRIwZ8dyaF3mvPNxP%o0q@Fg_?qP-G#37|N-TjBm8iI`ehpF_Dlo zS88E8rM%h?DJ9qIwsNYh-xGPa=c0`KWvGBI_=$EWbisFAAzw7VWOUPMzy!wXJ4v{D zc+Jc1`qFt7CrHcPOVLjWb!7Llk%|Og#+Y0dng>QAWQEjhwpE)P(9(`oA&~9ez z((Igj#f^z%ka8#%4@)is6BnV3eaMAV{qss&?<##>8wdxWON|dw`95)6%Le_(s|&t zm?2mI;r+(OZRMWh2l}o}PmUfNSgO^b4d_{8LU@zB(Hd%5W|ZlRr!CbfFpgWnIk^xPCb(lh<1on@o(piXnki}iYS1%mTv z9Vs9va~eigPM-0ptlPV|crO4)&Y$+gA}QMj^IRz6m&_IQFfdq2sQ=%P7 z=Q<^HaRBuN-0QX+m1D7*ISrW+#7k0%Bwu2=R^sLZPn76b=Sw@E4g?zQXu!NdbXGW^ z=|RL2YS-O1Q<5&N$)sq7Ylo~*J{!Vthti#-gOZ7|!FvE>X%8b54Z6v9dDHWe7is4; zS#I{R0|ZQxno=iR@sA08%Fhi5vMOi*7Cl1>^m<Wz|lDHjmOl{6Grr7C~l>(gqUagWUhm5AFr4~ zXA)oYeaoBJG9E9Xm_p`$EvA4liJUwns|WD&U_Oi|rp`f>J*^I+p;d=pUJp5^f4E$^ z--=clP^+{PVO^(yTL$vns`NbY-0<|&*!b1!vqSUa(^Jp)ppn?=0vFtXNvd!P$3h*) z_zk;bo_b7*2O(hFc}hyC7AplfFvZ&g_nt0IuoO7)M44WfWWB338b-XoN|p= z+C~ZkDh3Z?2*oD%!ze|H6sZcu*fA@Hoq;-tkb{0d2LkT6;fu9_8kE-dTIhjJFK4|Z zJoT;ZHsvT||Bq42x5?QoL zvCX^$bo7C2AXy`Vr9Lggbv+UN^sp+%J3KOwl35a&@jg4QT3%M#jX^W9X5fS1s-db zi2`Npm@aC+$Wx`ozb5`_E$HB`#z0HfMLOF2TBcG_9NbF^1c3^Ijz8sV2|U3i5*yVzW6LTXwk6>R99!lR#aDg^!7ZwFN~BZ5oGV%S zwIy?JUj0Y*vZr&BcL?BdH`8(0bA0DT!p*H?GREY(`>N&%n=KZTwxEa!ht|Z(&3ML* zO+Aw33>DaOgPuY*@aJMPb9}I&arPu->PTu4TCk|9!dk@;)Wrv|sB~%GSm)YPl`1we zg=I9F!~>s2Mz3xT3+YZw^b1-bM%EP?InL0%rXiPE=L5+r$belJ5|GvW1GS5sh*n%G z8kIFGhrLi(KjK#gRdPd78Jf5;^yJ*a-00Bk@U?~ObEC5hGeh&&=8$M=ngy67$cChM zAUXR)A+WApzq&9!H8wpvDPr$}{z4%=D|Z*0t0(T(+UqA)TDzNKWFc5lF3S{Rgw5me zxdvQtHwAZyALS*eWNejd*A6?sd>5=>bOp;f z@(@L4CdTLH7skdXMv=p*L7btVom9#oh@xh8!h#YxEIUuBNV^NHYm6{73$Eh&vU{`f9B?gCMIOcAff65Pz|{*APtuN zl*DnUWSYes78d~77yumuYxjiuEO$p5j*D} zUclAk$Mcg%^l*{~`d9`i;{4Fm$k6P_!kNghN@=RZO zcgW)uzbh9Ro}PJfVdmP<+$exQJ3a$oD7_&S$Z5{-!o<+j)$2o7nM}tBDc9=Tk3MSr zigQV8!9DQd=HQqPk+_Xc3D+2&nOPXUIXpT;b4||nA{OOE&Cbt{?@@?+j++cRxByfn zp(m(?_n++%ccQ$7daautbS;jWaof{7?7P$rp0@xnEMOew$EU87s6073M^KdXVP9}y z!~mKDz{Bq$q!(1)9gm&E4TZ(_P6Nb~G^3qx0?ug@>cj?Q1l%lwlw5i%B1UY~-jIkND?(Cj#;VP0w(9Bh{WC>~*<&~%Yi zq*Z{REFYM)<(U}2GCMT;WHFOA%BU3D5RXrO2dcw&dTQcHnbF2*kU=;;WkJZNa=@kv z_Wd};lV`3(}+tAQIe-Tr3Wr<`A>Kr6&{w znhm|=nRjwd&tHSOGCVdhbaf6GASt6p!m%F+5E&bMjOfnI%OE*0Fk0HpkH@^_qygHX zcwYgEGT-#F?)0445FdD^61vM?d4tM(!zOrVIsjn3wk}SLcpnMaz#&vdo!!RC<1j}@ zRMPq4!tlb(#B~q@{^L`?rX^(5c5OLg6MG2;#`^(WpCKW&b~kC)=nnQ@-dJqb#kC=7 zmr)ZNGM0AY1|;R66Jk0-zWs`bx}BD_GL1Pk1RUBHYZ)*?5!OMqQKFzOhKgBbm3Orw z5SIj>F!S_IwpUerhp^8w0dij+ENfE=5GKY%leTQv#b168s|Xzz1?);gu(KQ^o*TWw zg%REcT~WF@4E?U){g^l-9ePyI>R{QtLR0f0{U=FMU{6qtr*aycC@m_eUtB$sEy5+8 z;7n8e7BZG}k~|=C!`l;oPzE@3!MlVsK@m%vM5RmLLF*Lz*ht|-I)2)eXxYn7P)yo2 z-oo+WHQf&7p8V)zS<)$85p(iTR(U=Vc%pznBi2`Sm|jOuEg!ekS?jAkK#^Kp1|*Cm zcf0A#1gyVSMhK&m(}@I0#6LNip-fv&CdS}uRc-3=12{&&DShwlb`xKwm&Cyddn*p( z8)y@&_j#+*?pYDw0tKAu%e#p)w3bJo_*yjc$|S|QKflH2v$Vx7XTqM1r3xmxxR9sM z(Vd7G->Au{7*TK@P!Ue^(8=mYa)?M`1kX6TNm$y3Ewdy$^xfCv`0&{+VJpdrcgO~Lu=DxI@kfLU2Q(J?T3=1@_bdIqXZhxQH>JPM83+QC@FMm zaC#8Xygeg5=V6bvFE^&b*jlZ(YK^-33rd}O(#;0P)2z$Nvt+d@x5F-(Q7+Djizq8S zqxCg-;p6x#p1cRoYW@P@?RP&EI;asS(b&oQhp&;zIBS(_%`Hi(n=oxdaX5+LmX{<3 z(r~klXA)V|`g8*l{YUNKmZUfk9?NTMgt%Hl_MF(^R@Morng|6|g}#>o zDukA8P_s&{IK3yCEzt3k>gi&4N1ZnFQN^MgMqE@BIYJWEgto?q8mgBJ5u56VO~;aR zYHjXN){bEB8Ojv_3}Su-d^9pLZS!vP73aa!pOG~Z#s64gP{iU|1{I2=SNU)<(NikA z+o;E5mSl=XO^UYDa5xHp5*rc}56%xrxvEr;1$37p7 zB!eLm5BilvJ-{N=Yow8qO+J7~Lki!5JBAGm))C9cMHd)#lY$be9!et6C^*SfH%C?Z zVMja51_>Pr46XcNJ7lC#pzGZ`%jK~4Hb;P@p27&ZH<(JEvqf$;KpkK9vGJTeo~nRhCa--@QKO zLmE`4B#YCqjK3fakGU>6XG(DwnPP1B{sep>;~`#aZ*)m0#ri@DpSHR^TAPJ`WnS2P zKz{`VJ7+`LWbo8J70OtS;gT=6n6m|A;jvHvJ!2zsLE>5B`yy0a3Xrd*r2t#xBL=}{ zyKR=on=VI60mNG*YZ2Y!%~C zi1B2yQgNz)Ck-39x+KMSBn`U^1$vGIfh8r->5wD!tH1!$gm|w>G~UkY;@MXk0UwPk z$&pwg8UlIQc~0T`7+81k9~YfG|5UOYKTRPUIMS$it5Eu7M(0-=8LGxt+lUbAFY!oD zGne}06_ONv>eA1#<4*czzRWjAn^tBGi87ac4h9b`!IT}2AE#*MQips!3;YZB?1Y#! zNkJ!xPZH;ng`MJd@o}80TA>&Wnf6l8Dc<@Gz9}1vF7@Daw-e`k9_IyvRLd^j@B_>> z0!oUh@7U4 z4l2>{H=(6@8Cb~g?-gkZ6EGCq+gp(prZ_Hq?(fGjEBLMoH^9MLA30YPJ}^*tT?CNu z?+aL=#VvqQ5h(e6Dizb0TXi9R4ltH`9$D)CPzp!B!(NtOn57s6^q>UqK+(%7I>yIv zgAx|n6${9oQSSF-1G)CkdMHM9l#`AZA=Ht?P(jc;bL5p{;fdZ{EO0Yn@-e!y0+Jkt zbP@Xnn&_^|69+j1HO+^cHJh7Q>hY6J*`O{}vsZm-?$=VQDlvATDyxNTriXjNQzgEu z57;AsfgeG9s|2EFrHl#;;T3qWz#Fux%fXHbD{{sl&#vC%788!P)f;I<8Y9EqCxZ+r z$l{9Eor%y<{!Q_WowJ86G4#PWJD_!5zuJa)c_r;|Oa(S8DP=Rz6IRHRtEp08@crl@ z(*{d#Cn)5Q4(af_`GXn@Q-!&be*lPy)2LZAI=}RqoENuXn)w7Plzvl=H^{%BY{~q-3es zuv)8bthDk2u=~N98+WTXUW9PqI%Y&}1D_z>RVLfXP>?4tjviCSNACB^!F90gp@zW3 z{v8rTY(+T4VY!e2MRsKBG2gTe$UFyQfXJadt%)JPc{O^3ph4z#-8?P5F<2FDw`Fmk z=L*1d+1!91hDasb8EYpc~(TjrS? zI`2drTyy9GO_xH%m*h@9&*^D4IrN@UIQdOZ!r9M>7zWGFKkO@}$=Rz(br`fhb4WZE zpT6IMk6j-oAH7I;H@(e0L61unakqnKD>? zo|VYx5YbT<|GU0G7Q5u>(qX&uZmPipk4QibP!((q=+9x+k9`7|D>xPZKCnnT)6&>Q zPN8hj!c9E_X*O6t7|g87QC_xp0aF;qdt&RC6{dIYxeqEidM_%pyYm{)V2P?3uV$(< zlHLf!%Tax#bV9J|#xq(iGxBmjt5&cTQ7l^sZfDs-Z{-q7j04f&)fHFZ1Kz@0JwE4w zOouLB$Gk^OYanW=cSx(B!xco~C5);XEp?%`h3<6et;rNjao7$^i&p!3;MzqWsxY2M zTfJ<~ao6EYzs#HGNCSq|XTPH~m|lH7BaJ%9aX{>oGLJ-Zcwl0C@j?avRL=FE>OWf< zZfx!1vibFPrEmCH<DrCMsJ8_|M_wHR% zOqPr|HVD|fjE)ch)%u-Uvr*>}lM*kSn`s{BKlZn@xPtR;*S49ea{tklO5f4eF?_4z z*{q^{D_Q0R!hxn2_f1xqr&ZkmLTcKMGTM!bq_ywxyWw50%k3&WY}PO(j<&M7jVwFb zx^Q&mLM5x!(F-rpAXa zRJN(Yx1}pOpUI*Kp9=a+EEt*@m&UTrHg;5lJg+hTU}G%1hGz!PO*~KIS;y1H<9~0! zk9+XE5zm|Oycy41@bK>ir2C_H4`v@k_6lz$xI|AF!^B79=%-y-xsc<^rV zHw=4_@7vjOV`Bruf)Jjah;Oy4@*VtpYY|tbGA({HdUG4N$E!Gy&GW4-HkWV+Zk8$k z%`zd@$g{T0Jy{5Gl@KNrLre=?SqTR(mI-g+6>@R^?V*o+9i9`6+ai)MB)>-3T)Ybj zaFJ2`jbbH46WLDb3fn7nB24ORH|pD)OI4z5##4G20x>aRvTVmH!f6Slf>~?3B`3@g ziR-*y2sN%4PA>zw+sF+VV*V3mkUKFmxaFT^?oy;j*z1VgTiM*kxr!zdEZSkzM15#K z=ocp2+mN0*9e34L8mkp-t0q{pg@wEe_1XpnB7tEvoqdij<67IbYGo0sg-^9S??&}b zbwiu6#8bNKC+gKTk-W1&MpSdG=hNVvQuXf1$ zpz{ddhIrEHUqreu;{8|fK7o4H-#2L0X4yJ^{UN^n7d-cFD{p!`LJ*|&M&lWVLI4qp zio=Zf-NZpQ0M#ykZ*Q4ofoUUo5Qen)9lje0#C!mN>!6)LE0)Iv5hE(u>PCHT8*!I` z@G;zVdge%x$_$`gvYLqy5>AgEX&1k1o6oUvj zdk_~GZ8dlwfz*BJ*6>q|AI8Hn_#5+0#)ZEx^~nfv_$0j#?%GCUiTp%NKS<{T&4sDW zElh<5CqMpIVQ`T@1yxW=nkN6m`T!>$YyyKyHN)khu#jv*M#XM%uz}h#Bv7P@@^cQ- zq|&}C^+^pfjwNZ(0&Y7;OHhW%pYof+A-oN@*VZeWSXu&Rn+!l8iZFn~eQ_n5@YF;$ z1>8V2VrdmwotGI6B%cbL7~>wuVcZX4x6o3WjTSC=hme5>^7p{-2|~WIMCBJl#?rIw zd;ijK)}DtvxPy87nZfKs2><@Eq3lJ3Z~NPW+13vaXaD%04Q6eGKl2Sk*+&pR`fGz3 z>CE>v2c-=DL#cs+j|((4$U59k6%ycw%cU>wj%6Rg^HDtfyI1~&Y>V(?i2FDm(h&ZA z62G7P80Zwj5B?nFON2l0GlSX72>&;PABMd6pYT2Ve#o78|1{!n{5Uk>F#9dM z|2@1@p8V&?|2e#W0P+48m;XQGJInn9%3=O>r29PHe;MHy5gxbvKZo~M@c!edcclrs z$olP%?PFU&uQJet3?@*>4y64ewq?s34G|(&H?~^@j#2`^&yg4rSQHLz?J24Uzfz8~FK%{qC0E^l#ylA3zn z#SKb4yP=Gu{in`?jiN0grJz1hyBGottRLc$NV8e?;&Z?;JRigJaXg>ILwwu8c>NN7 zzl7)0cs|45cgC_85dT~F{aHMp!$Usyd4yYt|02Ry@VxMW;p|KNh9`SJa1_s5@w^?+ zyYN)-9LK{y054lY^Er4c7=P}vWxVdTMIRf@z?T?bTO}MQXSOqZ0&m-bPz|28uDT@m znNi5cX?-%7r0AC}fxqFoi020wsyHvS`688B{JN+LgvM1HtNZ~$0~ge8RG=P+x}HlY zz=8F}X5Uc=kN7(ZOv6yttBXxnf$<$|f#;Kp*dmguqcGC@>$prWF4KD%D=jER z@WV0bi;hM}@4P;007`_d1XIn>COiKFW7)rX3Hm6)2me>lIfVBhoIv>f2{E>e!m~9|jLb!$S>6bw-5uQi5gYcgr-3tic@XMgH2tR@N4}=1@s5ucOv`*!t;p#6vAIX_)7?X80lX^`1QX!nEg7!hkgxo4&epV|2c$z z4&iSj{L4uH1%&@M-v0pM-$m$uuv}-0%MImkxG#k9&_?U0E#&)hRZtFmR7e#==zMD# zq$kK%4Wenb3KI#_t~MD9+=BWbVPoyoNxnDMfEPR2CaOSEU^!{sUFfPSczFLT^;L$n zgkuiZ>cTVFq}H?4tSQ%N972pC#tN*N*g)ZcZ0s(qQ)`BEM3|sXu&x?p(`^}k)}KB4 z*Jpor>|gxk<182Q`s439F*5(h_q}b9Z!drOUkobKray@JJSo$@P?2LZ1?iV+BtlI8 zgJao`{W^FK!rtE;lsNwLTX;YEX^aKJ5B$SC{?Ft0zeoCWzcHBoFdqIO}aR`LeVgzNei1!VHM*X7STlQ zT!`d%Oh$n%4&xQZ-A|jGD3nl#7sK>~>`2X*L|+9&6=B$F&Q5Ku%4Ms~#%7-K!jZR0 zr36Ba94KUDn=M>RW8|Q|vF$ zbxRe~>YZ9+8^}W|I%rWxM|4c!(U;fFREQ|6JMIEyKUrFUlN#&n)sMX}FSNxz0OBW5 zS`-tXi{9D7BzPK+q(L5>W;6UA9IwIrkGJae2HE9Pq3b-8k`YgMpT&dE7_a0#3 z+~_=yU&@9jrsp`o46o0QW;lmoX!fcC^XX^jpPqSkGhtMUV7;Z zUzneN$I{XV-kE*xOqPB41CVd@w^|;2k`tTo=^P6vFuZL zUc&S1c=$It{e<8zY0!J{&hX0g^aQ{0_tf}Ab`{4|%seq9ark|8V)}}_;I|wo#r<22)bxBdF*=pu zz?R{m`7C?)DN&pmkFi058BZ^Co(hXHFm7x9eXnZa`tkN-W5-?e`_n5`px5#biX-}u~M z_I`v*pB>CTfbhN0$9@#yFus2n;g2BxBMAQRR(xlffAjx>y$9hRA>RMUWN722 zPP}YJ-%YGa!T2p2Atbjd&aB9|9?d+E-)RocR+xcdZFSYZRd=e(+lYoZDZB(6il%Mw zE^aPkNFfyb$?P~R0Q!wY#(WJ|A)|C~o$Y!JVw>;*%}HH=WL*^h&6-|p22R1&RGj>0 z52AK#SHL<56fU-LaTmP4F3RUFgjdGhWhDM~EyNiiEon3(2Y+Ia@-UQrv-p2VnM#I&@4>TCq9&=Ov|{@}jq~=%LGf zC~$MBLH=YqD)m=w2sA(JC?WRTS%f8OEes0Ur5zMb&-NjK+%Kwq1TwAZq9!C95f1=( zSJ|>YXfhR1Gmq(SQ4{PpSAWoYNn?(}@UGH#>7MZyd_5A*|m!l+7Ug?l%o(Hxd3dgij;;;5B zmJMYeK==o57|MPW;orvh4H6v9@o0;*qN-dni~;-aJ!r*UD4%)bh_dn0%N?i;ZCicT-Etn@8{ z63IN%Oal(Z0bO43`2|@I19bc3NvdqwiH9op^M}J(aiG$tHVF!`I6nx*7Qf%Et-vG- zET@u=vny5HH4=Rn457-% z) zgr_68>mItUKtW52al-HbSU@hMeFhr>Ix1w)u1c>qRzUA?Z8VdmMkZskXtwrsz6?aT z2~NDowz|cqy-u4Q^lq%V!tABCe1v}po9MfK0(Lam;BK}cf8zIXJZJG-#500t0?$02 z+jv&+Y~X3*c|V>H;-QVM{o{DY^FchcNz-0UTOQ*+gy%&(AH}n-2qZpQ(4jdCaoiNVGv{3v`X4?7r^OHl_EC2LrHu!&DWpgu}g##~#%J4%E zRfcEhFI1+A>ny~7!kg1X#SKKDN5cL4rHuTaSAzZL%)J`xp}xKH%Nm{>wmdI*_~p0Q zFAFxaR<@iq;P(SLIP1@_LWuvM+u?#Sh{ZE{A1~ggC+B8HhcjGv(hs>gs|+kxSj4?< zK;z3j-;#~|&7th~=B{Kvh=<{bh5r6L-ue733z;TP&thoT^3eO`p~mO`7W-pgUhouO zmcMVzJoIurETj+lJpaJ5|0)Y#8S--V>pMKr3+emke@)*%Z{zU_!&koi<^MqBK=_U4 z_50JKZ`C5+pzp6#*v`i}#&6BCFMlh3f3tph6U=vCr$4jr#BW1`?5{?@e+<8oP5yp7 z`u!8p?+-=4e=_?0SEAqlZS?z7_vB^%V)Xm%s0C$-uJ|^D)EhtaVXQA8d<7xnuKm=Q z_@I3hA-nlSgx`d4@UOz=i}0fe*}X3!d>7;YyRq!s5PlNjUqG0>IF@}of-QvKf$;MP zzYF2p?BFC)B( za5j55JBj~JXJ3!Doql-NLNYw$&2xm&1n8Tr`)mR`Yo6JNKpO~kVHR#dW=`{=7Em!;i7 zkD90lR&Fbu{m%>@rvMR$$6~f5?P+Iy*)gqd%mXB9?`RmBPh&uNxCm$b{Ls)JgX9cZl;NOO{T4BhsXdydSJ(Z4`t<)(VsKfsW5&+fEs7XCU8&3otRiw5?jkQRp23` z^=<+pI`;gsh;|T%Ip&llj-YMpD2dd8<#DV?8(16T7V$v47MjZM6;NkVK&5r0svJqp zp%~O3#?oPt(`kpn{&ZC5Jn@8mVa-ZmH%oBF>p8OzytdIJmNuEqW#icdeo?C8tmQl) zUD?DhLP_C4n#mlSLIHO1ZAE&{){w`LHgMkDkrMK<_`MsRetBXC3)l z4OFv=kZpIoATE$2lF||rR`IWM1Mit)iNooHz=^n5lkp&}?gK4a##>YPOD7e{;YkeY zgcRv!8@abp3h65GN&n4rj-dPvjMgIRtpit!b1bK){7(3@R6>E+tGqv^p348$q_h)~ zf<3u|@|%eB`JQv^k!%{Z4*|MUNY}}Gh(%M`)zWWR0YL6K{MVV&`@!C`l{H`(aWJo; zB8+3U@O>2RRPJ{l=IxPeM&8!&?;=WouN`t-#3(V1$13c#eAf019#Y79Chi)5D*xlY z16k6QdMDAMWhsTcZ51u7WKSTDSfgBHIC}{HRX}0V9!wzBlC-BOQe2i*A5w$QDDuW3EMEjb)=X z#I<-i_08bs$qs2-N}YwmS|T;;c49- z-MImMOy1|PQ)~;Ko>hrMLGxn1n4=rs)os|5p`BMPt$x~qkMlg%Mxc6UA$=5iW3kpA zYcvaqhhZC_J7y~zrax5SEBmJH$v?0h#wT@HWT6a+9iu;)X`8$Pw0dQGb(M?Q_#@CJ zYB-&-xqAcdMBbQLai2vB0CRY|$qS~efJSr5HwNI_L}QJx4BvR36&|TT4%7&s2``qRGjcj?jmB~>+=s&|#4|Amvzp)hvclhvUw)3;} zOT?Nko?pWAX*^%R^Cdj*{5h=E;<<=t2G0tf590X!3|Ai{Y(>v%qZ=S4jHKGjeU z3buV3=6yj2adf~79IEJ|g*37wQv5n#Kn(l$k;lJnnSo@Sem%|Yt$5Fam~oJHoR@+Q zsHdaX>iJjT&Vkhgln&b-J?fEu2&+ImZVe_OcfHR>u~rG^3}TcQkzIk!P1@#X!eFrr z<;uCBFyA$nQ5>;$7Ki?9H@RkE?#{K1U943u-hscl@85~+Q7t&3h@EbRkWe50m1@04 z_a>>+TYA0zOucc}+=B5f)T5osh)`H8LQh)R_$C?z&~hP>>#=G7`Hs!05roHaj7{_k z7{X46lh=Hi4Mk-0zJ5XP9D#(ajHx?O`FE6qb1BdgtfaxtHa;*-tEKFCVC$49NV^60 zs11-{TL}{w5m&L92g|oIXrWE+<2nnQ=7~lfz?D(yYzO;}@wn1#Jc9;r)v#r0q#@qw zVT}(k1l`e**V+@f!r2Pz*_FjR4Xhb*JlIw)ZN|8Q_A(V9n2+$7Z@Qr`PmN|qF0)7x znUgqd+M!iaywb+8jcg4DORlfM$g19msPARWpU>d=9G)-Wc^|m;iR|6NL%$!=?Fr}x zQ%Ls=xciChJ>cSSCzkm63~Vc#+pcGmAiVS7eK%13{Q5TNdo7!TvnamF>NdVhh%_z! zKsvZn>cThDH|;0$NhI=a!;mlQBi_(EC=Esyd`9;anDqZqHpb9sH# z*Ydb`qfEAS45e{_l@c~(4Xpf1$al8t-PSjTv^NhSTthgAd|L>oAtkaeQ|LR_HbxLn zAxg`qtf*3;)!t?B8odpw6Vs~2_G^0@=##!{ zyHtPE7;j@5Hv|aqZ^pt&gad%r3jW=-`uWEHjzj9@cm79xP-B=%Q(nuiK@#5-d7RoK z|EPmFb z_kYmzU;MUVXc)uU0w6Pi|M;ix!+-;0hJbscN&I~FB>EV~O(N|SV&;*07GJ%LDYSMH z@w3oyhNCj(@y>Ytd&50%y!TCSe#=|mcI4~c{`KGRjqmuTKljb={PW-Pt?&A_zwqtf z@txoG-Ie=#?tkFu_w@E1d+?#-C;A6Yo;rQz?74>@IsfQ|$1YyF{P^I|m0{d+boJW! zyWcZ0IW;}=y|Z)k*Ppm?^U3eKb^EFJE?|ES^1cP$*U@a-B7e(nz>-OC@d@L3ICcwEEm8#EmJ zfBvnu=kI9v!t)kRX*gK1_y72BwBFy)F#Bl>H#K~D$ii>Y@P$A4*INItYWTuWYnVN2 z@z?DAK70SqU(|N|6AfSfguVYI4PRKa@S?@P%fdhWeaScYTNeHu3x7nzm)9&DwfF}# z%-(3>@BE(j>sK@!{26=yAq}%74PU-u?+@7bH)%Nd`@gICe^bNk6B@q!AkS@<{6TwfS@>QH$1Oao z;mhA`?_Z~3_MiTxl%M@83;(f(gMVMcm;aW%f7rhNkcD*%Z`t>28oqqa-tV{X-(=yJ zKd<%tI}Hc_xrQ(N=Ksgumq%08#eX03P-IM#j1fghkuKkxiljoJR5COwluA^}MJb}B z!8}zejc5>E_gorC%9uH0rp)u4cb|Jt&-1*k-?P@c-nHJp-sdcQeD-J0`@8qq=j^-p zCEeeU{64AQBJl!=CrET;Bh7-1*cy^+kh(1CE@&p9R z@-jnxHX^>9cI9U>G|yjlGfZj=@}K0`oAd(K?UYpS5!FC^)a=my*YF&Yg8MHJ+;%yA zZaR4X;_C9XSJcO-%<}Y)Z()bhe#5pjls>Jd;rMX|?7p*HAxRy@&lL784`Ko@=S$HK zYgZt?T<=;<7Hm4{@;Jx_`KLZtKC|Qy82j1>Y}e=MbMnFe%6AE zsD68!JZ3y8g1hIrXNZJ>K9*b@lDM`QY)jkQ+@p|ge&gx(>S8d+4pbP@9;2X7E;Dwx zl|b!alVQn`w?g3IKDX+o5dK2|s z;4Nwo5ys*nhgZj8O%C`Hdl(9;m#92Fm+4 zRD-=ttiZ-ssC*)x!siRCLAF3>L3W@L;;tF?HZ}0e=kx8%m#Fx~ZF%lK zU50c8*@%PRf!n?9wt3ea)V|X~jh)}&QDE0n=SJkecg52Vw_0epb+uk#HV3MobtNx` zYhhesw%V>2R8+o`vD#wYE)kZ=CiZvA#tl{$5tuS9<&rCevSs%QOpBDN1GrUi zT&y)j_9Y^ldx+8qXA)JepPY%(r|pbaIMWFCB~mqhCT~LJS+OAIS{1%6+{|o<+b9!D zZhFQmS%uSrQmb!Fugk**L(C@@{56j$YL zTTD4DiTv~a@QW$It+}(-&h|#}AA2<_xwHh=43(~LbQQ|Pdav^@$SB6O?&~}A-O@qj zi+|d0QjDA3e)UVhQx$~|ai&%l;abcG2IWSme92}dE1Zh(dl!PWKU&D4^g&Nxq7YA9 zD}X(DvINCn6|^I$5KpM{G+Uap7KI-Sl9w*T*EEag9(SUk`fxk{$gTj-+WJBEl%zhY zKYfAo{$FwCRXQVBhyd!e!$)V=VH*L^B`CDxn)j-?^b&redDaeX6OGEE{16OSSVx6&1c~D$j*&P@;s}YuBo2`{Na6sA z{Ur90*h^v$iQOc2k=RLM2Z=vPY$vge#8whpNNgtY2Z>E2Hj>ytVm*m!Fr6iWHkycD%5s8H)7LfRr#4jY~llYm$JQ6>Vm`h>~i62S) zK;nB6-;tP2Vip@Q5qA4CXOYi|BvMJlNMs6=I*C*gF%p?Vq)sB0M2tk{Oj0M2N+L!g za|WrCNF@;?kts;(BvMJlNMs6-I*C*gF%p^lq)sB0M2tixAE}c_B@rW$$xG@aQc1)} zWb%+YiBu9X5}DkjP9l{=j6^0Esgp=05hIbwN$Mn0NyJEGa*#TSR1z@~ndIjdX(Uqr z@j1r-X;fgl(Y#d5<-p28V^}~yP9M3|FIlb)(t}#{K^6jtGwtIT06ME@Zl`}j^C8Us zu1?~5@PDrCfGtAvDUA7f^AS@hQ;ns|IHLI*rZM|dxH+iuKV8~+2F+(Nk$s$A+kj!3 zWT{+-=FezCxz)}ZIwc�(#N>2s^eX!_Wo-OUHi=u2({wDYQyyH+T=$UHZIb6`|LE z6|jT4r|TWHY}X(@+L1YI4-3T7z8z37LA-uE?vNvxX@y8_uwRaNd`xiFUf8b{mgewF z6Y<_s|4sWri{XoNZbkD?EIi=~;{dD~UOq3_4$X%#DtENTA=pLt({kExhIsN++8rmT zU7&e9o`=Y9{M9;%BXEIs_rlk^YKSw_#4b4l{l{4Tb@ymCE_ovQxZA}OO<*feQSvHlT%K9 zJPFPFi(YQIN+tYn%k_4LJ;&gvb|~?D!v8^T)hXx}-QJc2M1ClIKK-X)iN8XKu?>-* z4f9$adw?6?T3Wz6BK+lF=KDQCLu`gpbtsyjVmn@DEj$e$IA6Wj#Uv4zpy*bd205w4 zXZ62qKwS3r%j;(#W8YF^nJh!ZOS4y+o`vF={-5gwi1=@2Gq}CLgR%UwW-DP|FK=yP zcmY1TM`VW@(f;9-dXICUu^=Q??jlkD@oA?u&x4oz`{xrDggq(p&Fwf383RWG4)78A zyRd|D&l{qpG93M^2>S~O)m}ye)nHS*&lFX}DUoF-Y4D9VKiG2yVIRU;DMd8!uqm)! zv}Ot7lP|BT`+%p+k`p@FME&Y29KGTLS^G0S3v#O6@00}p|Ra=>Oeo#MOR&W6lIu`1O zY(w*Z?6Pv{(u<&zG5T;vyaM9djM9LMur{yeNM|;oI~rEBT!i=iPu5ExBkIHOW$jvj zP=2iCIrj*0!Ud;9Tf9Hyzu9gTHIJ~LKj#bi*Ylp9!cEgY?ut)9^Cv9YxWM8RzF+Z* z^Z0yXzEOX*c*q?uJRMtlZE^$RK?h4C+;Oj+#k(n7H9bTT8yT@)Q0` zxej+7!?_>Mnt0exaIeqXuOlv$`;57am_ODltXpvmH=BN3uxvTOd*|*+J&H^C&NN{> zlSABINORm3=a#s$E@dr|KYn=*LsvY&eXighIf55%{~F+ee;;G0seM87VJ!Svc#$)H zVav;o{$s>^&s@oVt}`AwLKDr7K=XU7jiKmp1W$T#`N5WCQN(pUTOJ<9Q+Ek-3#p;` zE~dLRvDXPd9r*10qlHBM=5BJ(b;5UDt9)|Ch?ox+9-K!zgdd>{lu&hv_SkT@Fy|nC zPRFk*v~v;S{L_B34&p8cOEr}q6Z3uStoHT;_^+7PCgM}Xe0GjhW9)wXT)Nr4jE8Fw z*DVQZ*@q99v?V28}S{sX`3AIFDo~Hh&_&;m$1Te^PBc~-c<$!kVYScD^8LH;%irHV zIAVaFCox%tje|D0gc3hbjT(_Z^Q$}kcHl0*Dzq<}5YH1-o&93l@dbHiQ}K6}5%EbM zPO!qGUYbNWPZ9BjtZ~w?#C^8A&3NQaJFuf-6MpARP=%om(Z2D*(!(bBF>U!pSH=kYw_jE!Zj6`c z4GF&6PQ-7Y>!-O6UzW}z9xO-b)XGI>Yw+~+HT|pP37z)Nvyy_ltNvUmzps#vc=lBZ z9h}RP-}pxzt31zx%{1}UXtTuytI_irMicFjP{uFnZv9?ylPC|idak4t{!8_WmTid? z!T-Fd#*!n%drRheHd5EI5nIbf+8UBmNp8SKsy-W;t4Xd$a$Ppkbl8ZkB)K-pmy=wR zjZ_UbGSx}0M)Ds>$=OZ~U8!;X>(zr;@Npg(j(-Q3Vm|`P! zlH}tgA7djn%0}8S$%jZjz(#668=1W%?;&{?$vfGIb+D1vPVzRAx3H1g%tmGt$s0*t z&qi7u8?joF*O0uL`Nb-9mzeDl_He&H?q{WgvhU8Ifq(-ul8BX#rlHXz@?Is(s8zjF@ z@(_~WW!oL|4jZW?(h^Abc+xIOq{gv1Etb?{NS#D#H0d5ix|2waWOG^ssfUv~iPSJs zzfHQ6NWDe6-((~62C0)s4JGyKq#i=*B+{;t?pH~75~;y#P75OSE2K^$HIUQ;NOuyc zmr3_aY-IYAI*HVaq<(?a{YafenlI_@L%NelrLj59o7B&fI*HVCq`MdCP9pUzo72va z`e{-pk?Kk69;AMXjZ_k8?xg!kHew`FPp~=dIH|jlI*HU{r2A3QokXfDo6}rK-I>%$ zq#hyl!=yWjR43B?5F42XNu5OM0aD*j>ibBYMA}}`-H~)Bk?O$aG<#CFBXts~dq~}u zbSIIzn{?mBM&?dZCy{DH>N`k%JE@aMvnJiGNOuycmTXSjM(SHhokXey>26NClSnmV zbJ`YC-%RQxQa6#hDXDK{Bb7v&3F*FpjTni)@Be?!3x0DZhP^ScX^+Bs<%#DVES>|) zHBhg4?YzB(cs?R{k-_2WD@l%g#PbZX2d~^9B4>}urg_Bk3YN@`eK)xKWwFx*CF1!9 zJH$n6FgV!qJi}n@%)3m;bZn2 z2u#cnWt`nia7=(YXy7mMI`Z=b@w|lD3o?fc979IxZb}o+uUPmDY{cLW|H*yrZ&n^hpiOYmPasT>d}B3mZv z!YaR#FvbbNZ>&!GS|}3wj#*4jFn2P%dUg*h{csT)7Z{%^*BQI5Md&|7sodc9%(-FM zWeLGK#p}62?w1ql z!l$HT4G~^dhA9Z4^NcN>p0oHJS^F6fU>f)B%0pIrAD5S%2`%f6rP{lR6aF_T*3X2= z{_~%P%USiMtrRZ=sX}iXcv@KgOO>g@P|#5-bs>gjKcy}8d2Vm4I-+_rNSxt=ASudzC*5E1A(xK=jSiFkj44bSNnfjOc(DosOK@k`7z z7X{wfiFPT53ohXo>HAe406sQ9ue!S_hpi$RYakJTp`R(suC z$P|O@)!vqY4J!$~P)1f9N+U%-_J3|C@+T@s69>2O?Wzh~EPIrfFBFG_!|8&)yIK60 zf`$Zmndt0P7-qHCJH-$Q*poiL#bMn>B0g!QUJ2;UHdv~x$;z*xGF1}9ee`0NY+>2& zgT?WZK+`<)G=s|WFRqHshAZJIeZNv!{pI2k``O@e!%79E9&QfkMPs|4=$kfbqqV8L!Ap=yUzNaS7p_w@T&N^@GEg{ zOJzE%Jx}WO&IM!Jvc{9_bbyoj&*X5Uof(u&tnfY{s{n(_ z(P0e_S?L8)HzCBqW1py8QHARFVG+Fb5m!^KCdMBcr@7}M@H@?rG3r+*xSB=$BG|oZ=KIx>OA*JywiYe|t?X4h zC-<@JhuacUg7Y?}UNS;_g#Qy(8cMMJ{oBm;#jNr^wBDfv16#8@158-?UAW^i(xsWR z1jJeP8)oxH3AipC%vBa6@`s7BRdwY;y@d8-IhxL^X&nCb&T9wb}TG1svP%WfAM0*t=4@**IOm;QhQZmVd8r z%Vy&l4`+^j-M5+0waca@@q^hZ;^lLR_f6Q(vI0q5D_PpDDq54^vE^}+xSPYg9P^~L z1h=R-Dv76KU%MBrUPW-uN|40GTJ5fAJZ9-0l`@jJ_Q6FfTN7F7DOL4I;G09ES0A}d zCH$9E;S#v+BhBeG7YqsRU42ag*L?3j+wijl!BuMPB=F$3&t}+6jM5P=s#zw158SwV zcd3jXp?iLx7RQU!zxnOSRwKA{ZGkvG;1l2T`Wx}S2@9-!E{^lz{iQJ#CInZj^A^Wn zitE|xn-TA0u&;Gy;<$o`-Qmh>tn^*$WyJBs5dXzX*2@z5%!VE@TkPRz8HH~&+xWud#<$YqtgnqJVnHb)%cKqrkMOJzh zO|!)C-Z6#l>8BeA-Q`D}D1KY5Q?TBLRsS5#&qeVR>xiE3uB`gG(d;dXTUd$g89vEM zPqD>J6c-gw<9&CbpQw-278z0e9iywxDW4Ud+S((6n?BO*_FAn)#Gl`ai{R9(#Z%mC zSoLY$c1;Aob-!%oJSmnvq_iCs!N2hSd=hWTYOhu8Ac8Bow~VKxs1xDw_E{o0*LkDK z6M8KF{-yXVJWp7li|4^cLJ#_8Hw(X7yGiJa39EiX%BF>J{Y$0AOnh|`m#(O5NAtZZ8FOJ3AzRS$ScKx&8$ z(M}NOw_KigXV8@3^^Ma4_>Il#DP?z9?fJSXP5`HlrrAw>WVKh|4-mj-Yew%rWYtWR zXGe1#KmM-uPTjFdR{JQmc=O{Ib2h#S^JJB;w?&2@FE4)bao&RkMEtK?k+pi(OD{V# zQ-|QXZQgwNrkI>lwsY1Ke7sGD509`8|333AEBu>woEPVKVaUro#mfKjpLV=>>)wQ+ zmK0WcsvXlj`0W)+{ByEc{ja_wjt6%u5*w>4VucUw1Rh*(-u11|5?SL3wX2RBuelnh zIE#mPpN(~Pd2{2!rlH?CiU1C?2qvL@L}1vSpPH! zzDW7_gPDe`{NC@6CTo1# z-3h-K77tVTg>hE?EW5m?7~d(@>*^<1^|7uSpJZ$Tbx?z$bAsM?=`2{Bt zUv7{01mn%2TjgJ@SoZU<7awO>McnzA8_wz<7y72h7$-w6aZ%e?>F?-=F~*#MTesp1 zSoWzn;62LluMgN;Ho%I%a{wP)(0@{aqnKn z!yoNe1jd&V{Me9p5953d!^%dMRbO+4y}KEu?}|Q0rm@N+GBVxC@Vyu{ee4a(zDGyA zI~XqCbwwsLS@9K)PPa3Z?*!g&abe{@e$2a#5z+D^y0?mD4~NF5TNtM8-8LFb*7znn z;oZ#0k8^up_)~^RKX+ofiShJQyWxTTtp1=eIo-%uG&eOyD~DCS+)3{S#@9rt^^ap& z_qX`*={iQaeWJc_9qWD(KQUdy_&g!HUf)ZQh>tcoUCD5-6W@H9#z%1K)N~mm;@PSn zcbP2v@12@1VQ_pMf4eGcCCmS>=|aX4DT-i=C#yZMU(=r%{eElREVr`8BigU&9LC$x z#)}R6SoI-0J)On4FJ@f9Jjf~^b$U9DVRxudXb}%9KeXxT2aK7*SGR3@#Ts8w7XH3w zzj@(U>ir(Nx9_`s|J`YWy)Un*8k@hu`PMiYtyM3>**@8yOCbJVv3w#)e|9gVeR&h9 z;t)k~pm`WXF6{+d{$1PrLfq&VyjFbCfs7`@lW0Jbg`w5B+CwknUJu9~_w#R-_Ob6==`+XsBZ>MQ)WMbICmAJV*3+6R<lhE-M|<-1qo{`E)tp|YJCu{_h4;5_|f*P0XJXQdw2#{iYx8&eJWc5nMx`!S&<*#o~S_T~lpo%o#$lUS^Q= zyt5zt6uzyWmFGs0Ow$zn#XA5$d^L*?5BgIyD>h9YlN$gp?m1UmqXQ|^v-hR)QwAW> zX5|Gr7BF1&_3fb3T`j=Do`1_A z6wrFO&qQsYY{#U#)*K#$Q-S-6Om9Y03MpBy?)VQvuVcvTyIChFwI$`fs&@y$no%_1 z_&I{2th(}B6+Q@!hMmvD)}EsXeV>X*tr>*no`>nxo#B)ROJh!@jtxT7cAH{>7dPpK z2MX>~%^rg88<|cQPhF$?n#EWz(H?>)loV;k85(_7sBPXoiy????&^s?6ilyDezns0 zDAI38b1Z!uL7B=4D-{bKf>!CsT@AN4gI*NWGX;Z0{&oJyW7+LpB*PilawfBM8 z_+hZxmnyFPJ&KN3&2SfE4#UhD(+8~sj?UU>~<-;AJC$EzAPKN$rjQ-ABe z_S=-8b+2Y?6^z28rgp78-;U8&dPqh*>l+2QzOEk#zQD)21@WF@5Z6fxiwEa^cwwX$f<|)U1gNW63YSdA(u=c9Ll)*wz=c?~GllxT6oG~1yR!Qy;CS>A z$eW6#M{m9OJ!#UEWm=jh&`qzf(&6h(d zuUdI;x?G!tb-QUB=ug}zP9Z%tvT2i0ZPH%!naiCr{?68FM%^SFkMZ5Mx+0kF6~MPq zjc*E~&zxSXLyxBS2MFg+1#?DD0A`+b_-M4N)P zx~?l(3$N2JYE^T!Bu#XZon0O_IR(2+=a_v- zyiQk17?Lnj`USTo&iE+54xOV%#;5XN~sFD zcjx)NUvOh_w3I@0EWPjGRkaNTzaZ$T=*xi{(RAgV4&C#|e?ju=d;WttF_f^3BRWeK zO~cx&LU>}r4GML=>#kGg(-4?Yq1I3mL6KlYKb<-|4FW583&q~X&?`b>eX<@*!@?Vh z1KT!SqV&04TasNe4Xvm5-#*#lNMYaqb=ey!G2OMl&m;Ve|NHy^-N)IN_ixlZ7})Av zVFh?RcRBx&C*ZLqn~PM@_N!Aaq1s&jsEt>rpW&^=>z>YUGtzk{o(<)6d@{Go}Lz>j;vfeGkosw zttnu;)dLs%i(1%}gZNefbYea_O>!&x^|3HyyZ`>noSt5heMs(3)vQm`WDLE)&yCJ; zLBAOHO!n_@-{^Cj-kiVv8&KJG)kEg+cNxN&Z13SzHvX;)He>(&y)W#y);mqQlX1!Z z(=+iu4gDaEhWs%RG7+*aT3)}28~W|DH@qW+euHfWMzdvH6FErqL6yP!z8bdozwKd1 z!uO{9@gEdSvssWXpA1sUL;jPiKTxoD&y~JACPT#1g+~rfc2h99IUA?%B*S~({g#^D z4HPU!^R#?ZGK`Cy37V8@rC`z`Z;TXDz&dz#T((0e1yjrwbl#f+LU(VJdZyM>uwqyD z(7P$H_uHuz9di+Ho;#V|lmZ%IQ**b>L!41}r%@pl1hTT^`4uZDm}9xi%zdd4bnv;w z?2jltuQ?7n_fny)J>wdEehme4cuU*+BNc=+W^AyFNAK-LDhoptUx2NZv@FcZrC_>O ztnhs=!0X((fxGaHg01Kj>$&#=&M%4Lujxne&5;t4ZGHhN8V1h|pMIub50p1-UGx$j z&t>|UD7~Oyd-QTI?tcjea(P$9^4cia3Fqq>i7#RQySOlD8v3v@azwMdC;xw2iVN403Sd8*(B~jq)@)n#d1-jrmvM($GYpr%Y zr$br5{8HX%)L)ca@4f4x!(n$h?Kihj{%1GmscB?FnbgCPJ3Yw0EF|+Ij%5N*MzcR2 z*hs;iy?Vlzk_neMUx%Nq=|J|!hbrAv*l5HI(4tC*8Koj z0LW!#qJ4;zTOYGde*ohr4M!I?^&c51yXa@zWpS z{sN7}(A!xQEIvQ~IW-3oB!pTurBV8g%ZK0lJIa)Q2cT!>+7>~Ajag@!^?wc zd@8E@!oi&jV`Vu4X-!2GjCnXgWqmG~N&I^GBmCG*qV|2dLtx`4kmEBq(^)l1!G0~nEv|fmq1Dpu zJMN+KY_x5R&;JD0di(WC`}&b?@oiWr4|=w{Jgl)ttW5Vf4*}@ z^*0c9K5ei7j;ed6)<`0K(Q@8>^9v!ye^fm4ET)fX%RU|5R0yJD>LPIHu+J|mNA_~pZq9S*Vvuq8F7RwWpFVaA zH?p@c2C-%1b7vnQ(o^72x?2plR%~;z)JFcd``^V{iec-kGt$p{(kYm1QDE=l5@@WI zNC|zjz4TIeu;|*Z!BzbFSfl9k{^?RU zvQ?zSC>Y(p3sA|^#0l0su|xBn6%ViNyz~un69SI#s3QM?1*6M9e}iDvC7w=; z(fupMxJ_wR8RX?07j!<&t&i#b$XULv49sC^6CdoO8Kz~0SYP9HU%rH@&U6`b&>0A16CX;&1{c*OnS z#yNTgtZn6TemITnyCD5VB1a|kUAwKub*~!T@2(|v8dbulTS1Q-&Z6)?0NIDX&KDtjOANhcWLpLFm|cdKBP_Du=qdNkhzA0vxc`9rUt)5^q|Ggqd!pnyLR5srPP9^nxM(B zXAK2A5Gjy(p%!>dp1!tyfbx6sqgi!+Ep&aZ@?3ZsmH&l;jG|;6Tygb26HtNr+f5bO zaN9a~;2Y6$=mWBMT=|#I{W>^pdx$x276R?9gwxYelX#8_L$2&d09{3_& zDONp3`s!yzq11XXTX!}6y2^ZgEN-e-e}6sbStQ%8+l|Jfro1dHupW$DH+)%RgXRwl zz=jvlBTiYyJ>Y=mu#aTvQADZ88!lN}Bpf5Dds!$r;AI0}IY&qKi z>2J&SmTIE%9GHqb^05J&%wGC)s7UBzi7#A+MH?Y_q2Q<9t;pV-ch&dXHp0Su#ShU= zBKnxevKRXvH3DN)`psHDl)uQupUg%Y!QGdx`QtmmFD`qtt_gU=WHaU}N$X>2mt^mR zHi6TxwVE?*QGV7t@rpGx!H)a-5(i5VH@|s)rp6Dj;alC_*Np04TGsZq_YYVYcsI#q z3bo(7nB%+oJ*m)z>ccZ&4fRkf3|*Z6wy+lU_nj^#NsLzD={suPK}C9U zmCaAdHfZ!tDZshV_`}C>VWw*vxPK`OhZm?m7KI*}o6`n6*Kbx+-h=WZvd(foIxk0W zXLg>e{ybDaLmJXv?Z8)PRy+9s?E{;7^-{H>9bC?t@D-m$@q3T%H(&7+mW-dW@#NyA zo}#YEo+G0rL z2(?ev5AQ?!r?1Y)-uVXYpI$7oAYB0Ac9SonHfSHU#N3zr?xTIw6=$EN^r3y!Ui^`x zdI+U@x2^I;`>AWE-kium`>8p@5+4X649*xTHAVZXf#c#tAlg^$9HTxz7wxMq-}z-Z z7s6|?tJhP|{_4B~+ssa&{nhU`c3V6|`>R9GG;gX#`>VNeQR+;DXHT0hS&jBtzq@-_ z_z>D>-BV!H9*Xu^t1V7@`xfo9zT+Btu@UXFrcOOG;zJnsq4Adr+HYNA&h>0#K@Y6O zo3}Zk{np;OdwMUT{nnCi4gKQLerqSM{WH_ierx~Bkyi@Pe(RCGm)H-q-};urr6a>= zzxCePKR)v!+|e8@Cz;a&x~u%0<WCqKJ9_NM_Rs0k5PVx$5s_S>;c2M z{mCEAaOyJ0r>KM;c;dVHuwq;fw8k&bx)+V&JwgAxAhHK$I<{X53hM#u zxR(P1H+z78k*LtNP{fmd7A0Nlfx6p^sG~v1ojcCTAg~9t9xWB|y3_-OHPJ397mz>N z>m#*3J;2Mcil56H*{?|rRmQ6aveM1Pm!Ix|papQw;8YLXj@{t3<^;0$_q@WCV<z+faGo+(K>h9%!tnwl3O)>_Q}y^MFYYxQlF)>0giR5zk3-H|zn; zmz20(gC2-k{CU!GHEQ23KRjQj2QJ$l{h_Aa1H$IPhx|2Aea!r}p=@anMCh&Xl~(Bi zlgb;SyBDGMUHWz4x?B(ZOn$Qnm+pZr!ZP=o=JtRh|NA4;l085>>Tp_2tOryj9)4ON z+yl|Cz-FOf4}2N;s3^sY`W2Kc=Hu*v?$5mr-BaB#y6W_cHzVD!N>XUm)}C%y6xzBq zx1}45p5JYnSBqBm)2UxfzIDUlIkJ70=zO*pTwEHIcikYP#eaA@y&JNQC7q8=?gpu) z>6_+0=mu)5UFy-8ZqP}X8=P>X8{(8-q(2Mjh8xz#5@Fum(7KM|xzoKH>~~X{qldde z#n$+vn|(LT*mP;{2di#)w$c6Az{YM^Ym@zc0=l6C+o4md-3{|wEMp^8y1~>?pg;qi z2dA`dASPa{8=S4)*0%F@L)v|f#^2KgV|v^dY8tzMS{RK7lyyN|_a@(2 zd0n8;w?EP)vkPR@jK1AX?t+1JmeVPByWs2g@uslbU7#wIu-Pi03-ZrA;`w>D3ugUr zF12#)0znHkpK$vwFqoWMoU*kGtZ%IgN?6wgN4H0xIJmM4tom9vaxLzHIR<#i$+=yi zm4D&uTfr{y(X<;cpXdZdRV~R+o#=i+3#-2Ry%V;)E$3JH+6j@Njn-j|PHd6gJgEbrP0l9cE&Kif|5s*7)Xy$OX|dE@ol z)t#_X^+V@%)lSIt`nF;fx=-bCCF-T{b;1UJ(>r5B9dJTtrkL1|4(R)AqBdUC0ltc7 zZl&NI(6ws!)wa7GaJ9_9OZ|KY@HDDjf3&>=LiX%FVWZIjdRvd()#2{|mjuT%YwCW2 z@wm3~iB~_tr1B~!?SkxSFN9rYxeM_JI_xjbIHn3FK>t0zczmy ze9#Voms(x3J=($4^r3R7Q9JkyIM3T7f$ryz6Xj(a+Mq8iDW?dXd*@$uZl$wF8%Rq} z>$OqQId?icbqxgCK=ERx>;ZJn-F{^i=k2#!;mkU!u)wZXxcOl8u-n2`C^N@4`q#IB zQF>A?C7}iKEtEF~*tdXZW$Avqg)N{eyQUqqiNu)eN2<^TV%iX@(;|3Kge$ znnBQS+p?s$KOpn&X-}`CKfpY8rR+YrA0WE(^l_7-CU`w@=drkV6I`1rNqwN&1cH`T zIjtp)&{s1c_X3@7R3TG%ws~PA#5~iuY|U(d%evAgm3tcCY1{KPtH8@ zZ8X+NQK|=qvM7(cYaOJd_^L=;ss)$f`%$NFe}{3d^eB2%4Ll0=D#^W74NK)2trmV& z(6KCBINPxj%&nJC$fD;MKE}3y?(Q-WI@Z|LcIz8h`#5AglP`sgS(eTVt`&p1TgTxA z#f8BCYetbx#upGhCenLv=_iN~bCxd)cn5oJUIYw9zDDmmC6+7QzfNI)Px=_i6X@jk zrZd@k0;vm7+3(MPuV1j(?{$ygh~CQowOT=XN2koNt`GQf*7X8UbE$sqQhgt9zjH-B zqwO}3?vUfQ0R6-P=^xBG-*4%Be;r8wpm-deGXL-rL`D!-5U|%C$n;Rc>;!**-sAWB zf(XW5r}3wvkAr7wIr7gW{h>94e_p*hZ1&&!z<=ul|E&-Fw?6RS`oMqd1OKfL{I@>v z-}=CR>jVF-5B#@2@Zb8tf9nJPtq=USKJefAz<=ul|E&-Fw?6RS`oMqd1OI=m50G!m zY5#a@{`1H!U;y z@dyr?N^napc!o3P1?)?ZNd#TZZ&uP$5Ai#97Hp6+4ujAAmleJ$Cg6#0>NUJX4nU@9 z%2sPvf4swz$iUy}pEr>RA=007FpQP{tPO^DSEKZWQ}FKhD1H9mS!dg$p20`YQNOat z7dQte_wTI#1AB2c`MpjeOJ(!lSV5LIodYH`)Gde@xVBF98Nm8YLQWM zieS-^hS;1aL(CiPG8sbu|Ni~W&mni>_W_B7-{1aQkQ=)l{%`h@ka_supcgb~ zPvA>ms?hICr4!|;pQCI*oJ)YNUYKx_;NX3F1D|oh`-u6rA3tlEIuxMZt z>wB(b5Az-ROV{d|j5zI^2kd>0tbItb+TX8`>DR*!55=4L%d=pOl;jZ_o%Ow8vJD@6 z|I#l7wM4!+@dWNB2&){3Vts#Ey~gk=GX8ql5*^~bbYl#jl1b}J)n$Ee8ZD*%)L-G< zcdp;>j=m4WC|IlDNC@lu*JN9^pZqJl+US#0n(_CbO6{ZdnmE??xYeiFUn2EF_;%1n zSVsB@p1gacAk2~VeE_mjj_!Zy=F4nm?N5lpl@=A<4Mv~CL6t@wK6jZ+pGok4T)*x4 zZznQ5{7ZP8OdtLweVoic{xA8*$@0U$lpiPS5B{b8aI*gKf2n_*Y(M;8+7Bn&AODy3 z$4UEvf3Y8&v_JS4`@>25g@3VMoV0)V7yHM_{saHge{iz@!N2rBob12wFZ~z)yQTl9 z|9bxnPxvSLKWSgRf2L39MF0Ow|0Dl|PW1o3^gqhaiXZj=zw|%qkI;$!Pe+KZ-aqS~ zrKA4;m;Oilv2@h`|I+_xf0mB=Kbe00pZ3GjQUCu-|HJ+Wo#_8${Plm@FQF6t|F7_W z*gq?N)c^kq|408}#gFqyG{*(f`TxnFRmG_4~f|x6>cv$DiT< z9Dn{yAO0A>2%Q+ee&-MV82?z||55%w$4^54XZ^t+<1Z`zf7Jh<<2Nh*f3zR`G5)jk zf3!dRasOcH|FEAw?>{X4ANB`-+`m}*KkOI&xc?D4asMORrx*UXe-ipX`_G^EUqb(9 z|NHa)&5HjY{rC6hLCY@dJatYy)^ zKM%uJHOThWp#1V(vfo&8JM}lBy4?Nt83Jb5l4le497~1ze%9&q?9H?3U>njC4j)*yo~fV8QS1 zuUm68kitF(g%i8^@L|)`)!XE`C!AP+t4#P3V;XtR2`8rcSiq%W#P@%(_a<;XZEyeg zPDK$ilMsatMUMGM>#NCBgpj#WMbl{@P6(Nf%pr3eLkGuv%%^YWnGPNE5aMJG4Q9{h zT5Ii8bpF5l|9kG&eLv6Zw|iZ^*Sgl)YhQa!d#%0qcWqm-u1P6nXIZ20No{A^igip% zDZ4jqbB%JHEY~g3Rl2>oG^G8lIdYv6UFExRHP6O3Uu}DuuS=q<%p5v1;=79*Z1ea! zB)UqlOOp*yGFu`tw&bbl#h6`yF3rL{F)_`LO3R9|v2pu85v8MtAk~yJth> zIwE??wyk3ioS!>Nt{bAK7&ft7c6M@-?RvgWh@MjB?~RAH8LqRP$kzqYS7u-NSX00C z0$Z^Th`wUzZhG&o&s@3ghrZG})wk)x?-S%YANoq!`0OQdm6L4a__`jYmC`pOlxf*Z zY%TaY9;KBZ`zDolowdaFEMK>yv@*5x`KkMwuCr~%*Xby&Y$_jWHs9%7@l2?3topRbwlZH= z!$7&CbFymHdJAoz@^v%}lwFUr4>jL2ORk$?pg0bDJz#R)Pqt#63-g++vOGJHAduIc1Cf z!tKo}t+sXM>r#|c&fOYT_R8l-Td@vBIc2BOR+}v$5w>F82_t3m0K-4$WW>pJCXAGT zTJwXfM*7QjC5)7vOYXEW3|nCP6JJN7yt1K8$MsWwiL@2#MwC}J&p-5++rZhjVx5Tc z_!?_B(})9!wnO>45apFi(|_>P+Z|~u)`6&?H0o+Nzj5e1+iQH?hYE^;*~U%>kIc3e z>pWCYo=^F+#jn0U%XJ+pDl5*-bL}u=w5?djp`vW}YXA9vp3jT^weWnA`X(FO)4u)n zNQQfP^ACX!^AV0=0b{~K@fq`BV{9NTAR)@IpiqBr9C=uXUv#9`@JM{7zF-sSSCsS8 zib_R(mt&5-N?da&%7g;86YD|P_@7wN|8+B5N5m%5`SM@Y3##Xk)i$O@h8de zvZ*wXewFj(cez!{?{ll)=~m?TxeI8|=Y{z6kK|SP9d4E3(yy|D^sC$~zQe7kl;7u8 zRLbvjD=HJkceoXm_R_CXexF-Wxm13STc!GaZk6xk_qJ84-{)58EyGo+-{)2-zt63x zl;7u8R65J=ajPsP{VEUhdA*{joFx4!?WJF3Y3WyaT7G9+rTTqtmHlM6%DU38@}c~` zw#seNuX3#Pt8|urmFoAoRm$&kD=K%$?`^A`B>gITOTS9>``jw!_qi37>i4--u9V-| zRv9V%D!WO)N;BzK`9^*}TV>3@{V3#fS_cXe+aaEs{-nQ0ZPPi<9-O?B zex)~#@7Alj;kFa(m(mX}#d_m+fEvA|rEaI#@z$<`{OhugN}6%M8F~0!eN(;tPX@I` zdP~pb&pOLO>dwx!e-3-hMvk4<70 z(wL)B;KKwP*qeA}i|Oz`*~}4Tj1*IA{0{QP`-$IOWJ^B$I&SRN8c6T%!F!J`vcq*| zRW`E0`FAeTs*&0b!9&AV+_ z&pLb6(K=~=u}eeW#c$Fy?x?(Rv8XxaGOP7+=9}i>mK_y?n@UaFD=c-Cd;Sfxmau0n zoBjyDn||fs|8^_R*Jj4gmlChC{>|%_8J>sy@zk}{>voMfj*Q=S@Nd{3pv^IVd5zW6 zysBn41M5Y{-q`eE$#qugmubV6XH~-Q-mAtecD%uEV6B%{BTybS8?QX~_y+5>Y}$+z=yJGZtE7-qu`N_?9SkQ?I#^vp*qr6V7o76Ca z#otZr5oFb%qjFO-xI#F7Prx98?)Sj& zEDxq!-DY!-{WQEwNa+vQ-}Y49m80*mG0R#2)8$FJH{UDJB|=$o&`zS0_WYNibuJ5Zco@xyX| zX4z!j7*;#KmvuXR^bgAI-7EUfWcoJmKN*fh{>Lu5y(KJx)ww&*DEdQtgxk$-c6})` z2@An@=h1#s2jpM1M1e;fOU5+Fq1z} z-<1=k=Gtv!Q!k~R?0>O7`hz3ZEx-Sb)pNY|=Ehj;pL98_lI1ow%5AIV%-8UHzIt-MY~uyH*z0xP=Qba04tpOBjnLoCR*hS2 ze&GoEr=)JPMvvUX#)c$S&eFy2zEd-Ml)Jo_O>*gR_JM-)(O5Lt)p9>;-QT}e*h}gk zHn{Izcz~VgG0I|S1^n(f<58(bj}NjlXPQm=?oX@>lN5AlmwgI*{1Myr zXKC4n(RTWSn9qrb?=F}2A*%sTm)ls}P+`tCNSVpzfv<{NG@hCM=k zOWM^Z^1Gv~RriC(49`=3e0pbb`Y7ua@X+4D4C8Yf!^%|Q z=X#G$F*A!-K^0n}erR-Ot_?ZOCJbu2Z}SuM-`Ty7?NQFKh>gB3fjKx{*xQ(g)6cN* zr053bpOIghX8Z4#KFebM=zFf(8Opz)^cC~Yvhd9h6P$NrorSqoyxvzj$I30*GVew! z^k>E&Hw{>Ijzt7rIcWY9?DHJoyJPk9EOSA#SAi>VJxD5b(`xB?Htpraxw%geJ~v^M zN4*P7nOo!Wldb5F!fw=yU2=g1EP9ncE&}~UnxWI6vT1BW!#?JNMk2qn-nWhIlE#i# z-nVbT2-L^q`u$o=OJiGJYsX#pf_-~O)cNsv8pAr6v${RP`6gdHckOc;i(T#Ec;YG$&na!2K=L+X!-7ulG$ zD>nzWM}Hq%Zq%W_E;6k_#gyBRe?WaWuwkdsCHDQ*-L;m?M|=KO|5d2}<6S~8*&j+~ zXI_UHlO=WDT@T)XyBK#^-Gtcw*Dm_;T zze;7G@T;6#TliJlTt3J5SDCt4_*J?M5Pp>zCc>}M^Zr@Bzsl^j!mlzcSol>c&4piO zZ2lR(ze;>kocjw%neol-Kkpy>=luiz{=$FWKlt|hE5)CGQ@bks{>y*fKM?O9sqar{ zkw1l>pB8`rg2D?we=VMW6#k$0557FV|IhmeMc@D8_2)nD9~8X*^Z)tx576F=Z10S% zrsw!BVnvjNJRJ`|?0dh_B>Db?7^fEsRx-v_tI#nxn2Gl+4t=N{fvTT?`kI8IdLAdkm5ZlL*%L@FJq6S^}`E3eYR?hsUN7Ozxz3rJs z234lBDP2E|I@RPd+i!4S^puz=RzJ#Ocinc|nNf`f0~Yp(V%A-pr%j%4nAr@{uQczz zFEjHQZfE%L7k2;pqw;rjd|2tzb?R+N-pbZk4~jp~W*=LZKfQ)((AiHWc_vIh)LQp}T^;S!Z?m+UV2xuE?;Rerp!9 zcOKt6^ty1B8D5+7`~B&2*kjYtI_}#Kv)Rr2z3x9}Iy2v3nOd>!36}kKkC{tOEPE8- zP=Di{-OQ)v49nfS#<1)I)-`6I+08s!^v%ePI>oAHu>qHIn-$pWcI>vUuKn5qJ5+n^ z`cz$TZ1#ZyeOjB1hB=pZ7x+Im-gVQh{dRU>+t{l^TmHq?WtBZYY3C-^s72K=o%Y{i z>6T50z3jb?1)IDYu&G)aTR84;gSbaYY~%4&cZ2$;F}Lrm<|N_^8CI`$rg|qFV|&+5 zu4%eEfoY~rW_v3fVK;pSw>B7YiFI6ZzK&~H@Ihf$HuKpuspo?%^{1EmK3 zI5>rwS+$wE$KX19*4nYV{iS_uo~CQ}KPp{gDIUXrKau%68})I`g88YJu)ba6>&8oV zu%>r(Tc*tTQ@&qyo%vUd-`#v>SV8`_<;(43z2jn=&I8@E6<>aNkHsl#tz(DXw+iB| z`YoXC+VC6fv}sz$eIFmNN|trj?+Cxf%Khfl?=99x+O)Gu+Ar}}Sk<@XwDVtPGMm5q zxO(D#zcP0wYq=zo88qK}ugUl6taWmm0e7+P&@$8G+u!fJ$n>XnezDT;E_>R_pw=Ff zyX`Oe@w~>e4>jM9pdV!ytEq=}Z z@M-mE+Oy{@$+$w99l5XB+PPP9jE(Rep4mN;Ox$zWrL!a2;TuHk*6=0`>i?d@#*hBB zK@EdfETmgf?>RMcSqo+3t+w@Fv6%GEjyf^9tj>hxbt^4+#pb@<(`2vL1NL&U+W2?D)a3PFL|9p|Gvw9_YI1gvC$;jjA8A!R}GVMdb?o17B@OxYxF#=8Ttj_qeyY ztn$?r1GjCgq^YSKiCmZaj@{3)nlSTI70o;M*`HcPd}On{%KICwuCDP2+vi$2L($xF zykjwJc`eQPka2+_imqnT%drk+{54d)NoYf2B=rax#~HI3+hJ8MM$TU)04Re!4n&Lh+#-{|pPvx0J{G^1hK>-}Z5k5*-x#}C}3ZPuWVb+z(&cD)j+*WVGo znjLg)c;NR-cT;9}czffm33I# zSDP7q?M}Vj33eW1M_nmh(*ehM+$8J3DXrII^U;CkKm4KE`+3pmJN6I7eX_##o-gxx zU$m)tfxVM{?6oc$k7OziAsI+B8%1-~mc0Z-{)Y{q2 zSsL}nRJ*l%Qr2#-+mR`$7rr~+^_bRUK{aj5pV}AL8}z*1wCv$u+uLtl;y91?CI$BD z+?>ARuwI!0dz&_ijOeESEM=NgoJW-=%e7;E4(QwHmn(LcU$%UngX_Tb#GA?OXJ}LQ z{+_&8D(#+?W|K^ z?@6^tm!lRrP;+)jPP8xJ&{##oO`N>T`UfJl&wnqHq6VqjKiphWm59X&b?ZGUM)mwfDv3u7%>p}As-^1QE zmnU@Ir#*P_@Zq;Zs}$I4zWw8?;lHcnsW0`{_l(KWS&sSz_J(%ccH`pM_jYPMvR-7r za^04{QeM0n(=gF|ruJIhpDjC8J!yArR;_J6EM3g{+`H5KQp~!P(3ZpdZ;bKM9&t4E zyxe1<-J6XWYn_jKv*kPHjQr((WQw7l%gx?zTWbC8e_r-f$H&fb<*gE9OcR4AleNO8RmL9hzzrfz5hbo2JnI28C z`zvH&&i%34@Cs4SYtKDmm+O!|uS)(R7F2h58^e$VDH9ijx@9%!rDf^CXYgSf0@+VTG6nkgB>zP48lSD8{Fzv-M2c?I^WdX*CzIbPPcJZvZ47hY6QZ`5$_c^OA{ z1kbYbucmWwL_e>u!Ur!LQ?JMM{&wYFv|2FwkEUORtLcdT{cC$={ubC*LjKyx{ADHN zubs?aRzm*T$^2y{MxV^ zx1c_hRDWgvP+)Hf{ezwCA6NDhCG`)o|1Ge$g#On~ z_P?xz{?|_SzpRA**G~4otc3oT$^N&%-je!Xd3`Iew}k7PoxHxW60UD{^7_U~xW3uR z>l-WK`o`q-t-#)ru5WVuP+)Hf;|C_k53GdogPj~duoA`(c5?i{N*F&dIesXxx1{ld z9G?}~Tf+E^$?+L0VSHvM$7igB@tK_*pRp3gXH1UI{!sg~lE!Cp{F|!AD`EVrhNqS= z{!NwR-_#Puzo~Nkn_9y7SF0XhQd7)d;Fa@>R8>*J{32D(FSI4hFH+_FLR-T8B2~^W zQcIX$XyyDu&NoY*Ue!TFUif78nOo3@1cn^w->v?a{nv~vEYEn)tqmGifP`DRJ; zH#t8pu(yQyu~yEHwI$4twQ_!}En$AFmGfh53G-vEoFB{l&1NOdkLCS`0((oi|Dcuk zAG9Uhf6&VN584v$KWOFs2W<)WAGGrRLqYzQbpJu#-zl)Sg!?;Md4ES+!u=hsyuYI@ z;r@)5`m2+7j-cY32PhZ3*|! zwDSI0LH?F>|4iPWEU>qP`;%IEe^Oh*{YkC7KdCL@{-jpkpVXFce^M*&PZro)()~&G z_iZXgzZNZjPk`TbX&m|Q+Z07o`efU*J>wOUYQlFTax${v@?NOpx)EW$gyBEA$)zZLqC#9lL)@eM`uq912ShoC2ln9!bn^;v_Vaf23hxva?&>wBTU1E5 zsGv?!{(bzydiq562p#3<=R@hb55$_(N*~2Z>8-R>zE_$lj*1KZXmj<8i170Fvk#5# zuXOf{?Bo*}ILfboV2E$%X!5&wMM#}nXjl}Cfv#_aUwBXN5q>_AT|0`f^m7gj@}pz>DbWzxnkhawPo5=kd*gh0ws3tY&jB7y zDIFU-HFjdGq8RH=jh#9vwyM6qlatewDO2?EcBDyF z9ebKt#!ipLUW#XcXFx!JfkBz*u&~i#Vfe@=;u|Ps!2fK>0kg0EEEV#(miT@>Sg z{EJh*{wuo2MCHq$j`FYl6;5H2^5w6T@~{1=`)#@>fddU;9({|N4CXs~UrUtpDn9O6I?K!ZMn%QZ)X*cD7$)87p7h!lk1_ zjUaa7q{(qprcRqa*}HH5frEdfXj2azK63Qf@e?Ocoj!B+-1!S>e_p(l{@3LzSFc^a zar4&gI~jNH-OqgR@b9cgkDok!_WVWm%U7>+a`W=vynXln!^cmb6`fMLditfy7#Nl< zXH>pIMdM19t5mI4y++MiCbdoL)U9V$zk#_$!$yrQziZOe%DP!|n-<@sL|15#ts@hWau!@lEZ(B88?2y|MdC) zPapr^&i_vK4vtRFox61H*4@RmM^Cq2z5BTL?bqL9z(A4zC5Qj7$p3hs&KM*0z*q?{ z9u;yQE#%IU?oQNTd^;nyi$d!U7{g8b(qPj9?tw}d{3(i+ba#cj72FQc_JP~6P@I55 z`niST?~;0V#J>r*daO5at9F|D2{+l(O1f!e=Puo3PmpwzopYs|>`9jHZg8KJZVVNb z4Cy92jfRW#$p(#dcZS?yL3Cj9V6Y8S4q-M_2H;=yTY9* z-DI2I2$2q*m!))5o^+9JDwiPXrgNMw-IV{^q*rrgUryxoIBar-Unk7^8bBAqW+7lfpVP#0XLRl_125Py)Ug$NHObq7Xk4 zseAE#)%3j)Hwftk$|H&+dtrv4YVZpp9EXlX3@l&>_Kbk2dev`@SfSj;k&4ov#<-hQj)99Z#GUSsV$WW79dJt5Bk#p%kq#(V3)RK{}az7AJQv-p0br_%Q3)`=1+9w$(qb8-C$Zm}2krc}N@ zW(cgIt@wTs$REm;0L2XZ4d=%fDSRE5@--0W8i<^v8eyi4fkYw&nm77`U+)v8jQr+t zMb3RIegwClEFoLJU0Mpa^h>zN(ZaQ~=>9=CN1E{l!%{lma9CSBjR<8l&MbnLsDf~6 zWx=o|1hvHr`-s+}wt}MJRBOr7NNzL5^y4{IJe_abQ8cZhc2IeXmi=`rruzM*^;*Ni zFJ};f^9+aI4`-s<{I!niYA8yG&XZb*g0n5YpEY!2q&|X|rf7MQ$P<5vX!Vp5^(A9* zG|CONKZ!s)CAWGGRKw|fLts1AC912%e=4RH?N3J-;?RNuulrntRSdbBScLD=+l z8lse`*CZ>y9sXtiF(t$y8=^`GqnZ|u+z2fwHR^AQ+e-N$>ZKQI?bqR>kP2OO#v&G# zgUC~ACv?4_E2uTU{)F(b5NIilFYQY0&(!{lNNKnPb>{c?I>9{{?#>wD&=?^IZW?dW z_#&FS2jaS~C~<}2BouOQgPX>fG_Fa7yNlADj}g+jejvYN__UBa2W~ZegWxZ2Q@Gt= z0gYu^!A)b3?r=N8JrE;38jpCw-Cd5?q6_I03h6h)?T$Yhf6e#qrW#q6szsQw2PDl2uY zVU3MI6M)XJ+PeW(@mF`O&v_?3!3Q-5C$Dx)i zu66%(O)VTo7K$^YI;LKS>1tE-I$Suc@IGH(j~n4Uie8VSa7L8pRKF=t=nVbgR`2Z= z4Hvy4#ipL56Ql;G1L~53Iv{9dWMWjtxQ(8Xfwi8lo{_P!K^+4fgh$78#(h8B!vJ-` z>7X7s7t{xnz|vqcSO(k%8i0F2LogLA3!VhafoY%-coi%UW`GsIEU+S&4bsZMZ@@~R zGOn{y8885=fW}}|uqIdyGy|)HmS9cL2CN0P15H2;SR3pDnu2a%9nb@;3wna}zyQz; z3<89c&2B1sj2hU}G=|{0>Y8n}FNErr=)C3QPs9!INNfFb!-2 zUIpockr`k+FbixCW`pzs&l?aANEBs!XGH@TfK-OYAkCR-g4C~=fzHgnGWip0>p!*z(i0N zTnp-fnd_VteQ+E6rNM(>8So@%0H%Y6pypR+r5u<6zY+KpEDyc`D}Z_vP#-{J&=@oY zD}kvUlnWxRs%I)b zGz0g7^}(ZH127FV2XBHFU>4XA%mEvLN-XLFXas%-)&!e?7GP7*2DAcgL2J+*Yz}&Y z--AJ5D=->t4aR|Oz`0;sa5>lxOa|M7yTBj7RImeh7PJGef(*2FbVYw9Y|GB4?K%Oc3w8hxf_C6Z&;d*bM}kj59duM5Kt0f4GU_o{6*L6Rz_Oqf zXau$cO+ZJm12`BQ2?l^V=+L4;JunU|1I`5v!R26CFc~xgcY!8gD%b%$3+kZ5y9w%n zSzsA32Q;LPFb?~JMxYT`6Ep!Wzz(1-sDqBt1vCUbz_Oq(Xat6VCSVNM0ZagO&{3`h z4Z&?-S@0le0-gjrfEl2U8R`{i2z~&|f(BC%4ps$qEMNy{2-<*UL0bw3yHL0#?4fYb zm%_m?(p$kE(u31U4}gFZ zfjS!G7x_U8q9g1ic7dHl7uZR3LpufaV$e=O6EF;nYd(zuw!!ZN z(luWXJP1FX=}B-JNMitfFdcqmAs-9S*dPP`HQ-Zl9{2{NYm?se&dN^E7~BNXHHNNj zrtr@MX$(;Yw1S_mVKkO70NcU;3+M=*1Kq&Q;9&3+7y!-(BfIJ9NY`PA4p>h8hadt zKM16;NFy)}{s1D*r!06Aeix9&8qJ9CM}RaYD+lJlp8zU9cUIN`M&K5(Cg=)UfV)8( z@E}NI1{$l_!atq#@Y9&43;Z-@pfR2i=ng-PJ!ou0V<1oXBS8)P=z5eO__u=5AdNj} zOjsU_gTE(8V<}5;F8m|FX!z@a%i;F|Y3x`5Ooks#mX9GTg1g{f2&RI)Kn435183o< z`CJD4*5Fn6!$Af1w*WHXj|RKI{{xr}|0K`=@hgEeMq=Ox=x2aNGigqSaOl4WYr;PU zOhWvIpauN?U?Th$pbh-PL0fP(NMpjLU>Epnu8^o^;wu9e;2!9RG5&i)njYVxi zH~6Q3gTW1805}Sa1eb$duwN}O7XCrtbod*9@$mbCiC_wt3?2dZg4@88;9l@5cm~V@ zPk?UN-voRE|6q{%_WPjTtj@||&=mXwYz3|dUBD%v2llHC`og~y38bZYKFV;q zZ$%s3%MqlrC%=eC&k|^(88vMIGJb%(&qVipXrnnYZFC=0?`&G2jJ$2gXxdd$y1?MB|qC17O1PbjA6JyJX}SgZVJ+_~ zQN6In2&yyUP~EWRPYy+Wp!zY8?@M||eto8TL2f7Lscu+f&)#rTeXvH&>&o49kB6R= zih5xUtqXTYLEnYvevpr^VD;M~Tf+jXGgQyXP4$ZEI=Lx* zs_&F<#m|N6zgl+Q++Z2^DoweGwC>1JQ1wE5EOM9L}RF5f-L>UD0Jng~ri27zq zmBvG&T~U8b^@eN`c2IvyDbux@`dX@Il#im#i1?z-2z?0OSCoxtC-hbT*-ZQPK}{}N zheR6??Y3||bKw1k=u1R-i*_Q)+n(oF;j%0oUp;>(J~LAL8z){D#C3=6zN_cliQ6a6 zxf5@@;+*LX2KAgBcwH`h4uy{=>=Wmp9@~xQl6q_xUQ*)NeYhPWzg^_9?zBABJ!;(KUGZbylS7$j_;8T1Cr% zo=~a1shU<#)LgYbcav_a&;7W)5wiCXC8zE)K%R>z6MMct?c*SiFIsHTeMCzXHrw-Z zQ1gP?ifCuu`2HdeJ$WI!DBVuH97G)PzLR?HG@cf5L`@cPy7O_ih~p;PtFVu*O(IUG zLgmqy*FB-PFBIRckiM5JOG-!d8sfP0bV;>AJnIsA@s5yi)6=NJ<2gDr(cX!opC1dV zwt3;4RJS+!c-8I8Mb~MCC-h^a&2;S$y_8R({z+)We|qw$ULVDE*_)>+#yMgXEUqu& zS(#{a^qz$^di*c@im?#ousRl^Cu!<6!k?cnjg_s@bB}~xv~}8Fb&F?iLQnryJw4z4 z;-*oa7@Jw6HHZjk~R=7nSQr$Fq74?rwNOjX22CCZ!`T50NSpOwHJug$^(9``dZpwF& zzCX`X;SRvnP1OhU`vO8w&*D_Q7(oktm^^oCSM=oZOB||aB953r2zL~(E5c1rv?&ga zOZ=pp#vc@_j@#%h5^~cxNz91E_({wHL?0|>9>OikOLbG*6K?viy6IgO)g8?9M2tnm zam5%&)Loi$S>xE0&tj~m>giffby^(`RRAM5^MdAZ9@{H==x@x+2Ca)Z$flFfU^UfmOSLlnzr#Y6I z59C(Qo#r1SWj;_k6hb|Km|v@Pg`UrUanpQI+*J_sDN){Ze}VK=u42w2%0tWt>AfPF z<4~M198sO`h>>#Pa}XNwzwkMTIp8Q>U&Y)_99N7hsV!3Pqh9aHPc`_9pX#}Y8^Dh* z+`+tV3wH!>Yr-ALui;`|D2{{O_xAm`ime!J-*9{PXB{bR+&!gRth)x~Ze^Bd+Vptg z|5oa^k=~-_^F6HJ0?)|lHTW%PsP>XRasOXhmU!~}ke|Ht?cm3jO8TtlmyfWib;{3l zJ^MzcbCmR5e%g0}mHVkxrK{m}Wqke}U$-lB&#(r04UF{{RF!(Zj@X;1K^NGZ8%-{k zKUqrZ(#I3+2miPYaE zeZqLZ43=#D>zi7`YfC-JH+wK7yIqzhyO&43K=ig{tMFM_p*;z%SP)fIhWIsdcK}dvXb$oMZQoi{(veauYa*7SIM^?RX@<@$C#b)T@s;WaYr zH#U^<`TA_T=7m3DdGXKM{nXu5#^?H{VYi>K_6zq1eQIwZeRr|UAZ2ypF?8@{iuKl9be8{Ev zR&P!}V=ZRYY8Pr^CGFvQ$6FT9nW4`g&G(m8>q8#J*OB{ZDW#K$oO0zb1m&T zGrfGRZhM2u(*AskA7*C$g2i|3G;3y5M;V{%)5=A>VAG?OYr878lJVd0_}f!oFhAen z+D4~jJBsCct16|lnReRn1KZYjlJVbC`~Yo}Y&Pm;nf+;bYW{P*g@2E1*4t_D-U?~m zW&C#(|7evTvsvSzcMbmR>Ll&qddKtgv)Qs$jl5PK?jqyAr}!?bc4o7gjk<@|>E$fr zbG_A^OWEv}dGoc6v()zcf#P3X_bQtW@i^yG>a%Q*v0Se_F@DJm($cq#J6}(x|B>Rq z@oV;yz35uoxBo(UUa?%CUZ?v@_Upyl70%C9+w~`k?@`PDB}@3dRg(oeYJ9HO8#d`B zn>6U$iK#l>Wcr^ee$CwFFIkJJyB^lHk>?f5^>gp;e#y=S_?W*M(pkn=G>C82B<&@; z)+;Nz?BDWyHC!K-p7oOXjhxtfOS2v_AGm(=1LYMve<|hBwh)cbr*eI^cgA*n_W(?Sp3?lant8Xdt$l%>HI;j*zA^7oi6?) z&o`Cp-Cu;gVhMeQcU{v*teUI}~UNQSN-@R+GShfod*S~4G^cAzaK5=czA8n=m zTyJ2pp zzGjo!oO=}DE&D?a*DIf_U$gTgt!f;uB+o0B>pc!&oyYAqxtk{COFL7!es8^=ui3II zv0XhbsP$Q&(oeM>_L^NiR3p87Sy^5huFw8E-V~if6Wp*zC1SX zlk8VhxjtybtkQkv3tX}kP2TazEXHNTx~j~cFz+qdyG^EqSQZ$U+S znLk`_^Lp=V*7kaGv&RE!3VkZqyFEM(`|AvPVi49y=#?^*{?&x^*DRn_yzBEfvOZ|I zezS7#HFLTgb}UFy?dSUCOaVNxIrU7*P60j~V1J z>m{uEfn(B64cEK1tCqtOEQ3;GuE~6h<@%(s`Z+A>#DhOI_hq|F<$6t+bq?E8sbA%% zZ)HEJ7*hIc+qKVOs~=2Af4Qr*I4=#?+n#aEVfnkO9FM)C)=#bv_|7ed8T9saJl#{a zi&Ub+4AGED7F4@{RiVMIc(ISo>qb0>h;Bl(m%OwPY$d7sgcXM z`3^FFxPEWtLpf~8pW|xYfA1pnv0R_(aXN>&gszz$zekQwQiVSH63Q{X_JyP+>iDEQ zrJodWGl#Wt?&?^ZNxg>a+jV=8!{V20O7V|S)8~5MkI!;grD};uV~pkaFqP}GX65BD zyH$-2*tC~+DitXGgb$x`m|oe0_qNB>`pNZ{-OA)L${xqlgobXUnTLWfgI1oa(%m-19Mr~ zlan_5xT%A%Gl}arr+MYF9(D4XPZ=cZe=65oZVt$0)n{~EYZ@%`EraWALPK-eye`IV zTi0_H=_r-R{w^k?a+%wnuFw1p>(wsp{CCB5H)Q%6u8(apIhQ>%Zxwgo z`>rCsC)X#;osr8zYtQ*BGgP+gSfS6DlgqO6Z&?2tC(}>j`Ye-0xhy^4mxP8!^17MI z^?I7+xvXyQP0>RqIEwT$xZcfeO)i^S{kRhGvs%8D$^KR@$*_OUu=-_FRQtK!xYd?i zwmEb159YIFIcm7x#b8%1`_X^)4~-0@ot|9Z<Xj;F|G_bT<+4p5re%KmRjofR0xs{LGl_NN!Q%=+!*OHD)7{O5X$2|2ke zI$^<^%6f9Vk-_yoiFOO}Vw8wd{I*wJU zk^PZBndh<3>#l6ovr_Y)>uv5g&SNv27Mcy2DBGik>#gjp^4R>Rx2Ljd@_OXS^-1wI zdF;T}z^*@5RIjI8fAoB-JoYfa>-B-()b`8uC-vIrvA452&5b`N>v<~IXEy7Y$Ih

-(_yX1!z zSC7d4O~dt1eO&X{x>pUW^{QM!m6pU24lHiPS}wt42UP7PhhOuTL?j;qul z`%Qy=^H}!#=j&E}Qtjt@W3zxfb|~yzy-PW=ztV8M&8d-jY~=F_rt`+hcIV0U%L795 z*oxm1_6~0=>uoI8$9@XWV~NY&lp6Oz>XW!$nK>$tHJy@MK5&s*f4E*#VQe0I;{DRT zaYtF-GPvGz`nWtc(BM?&xlFbHtV#Aye;b>}T5QRlk+?wW&A9%iS6m+JeQ$iVSywVe zd1<&l^3b$A)~koR;l`)3zIk$ebj_K0>~jB>dOnZT>nYb8`OnT{51Lx8{CQ2DNGFNw z9e!aQ9=r1hS>KdeWWQC< zWqIt3S#HXvkFwm&xZW{pWgdI6=v83I5E);?^~)Em&SMP|W-mC@QuZ63T<^AVZ5|ss z`=E1!elj0oxjuH^hCDW9+_IK7Z5nxA z8m_mE*@yb@_+-8OIBBOR*Cz}(n8#wqEw0o-tImtL-pWRs$JX19GJ1bcUPqF+-al-?;1PJhsVq z+NFeka$IM|_3od~=CS?1t+n~iMIE>8nvdyutpD$~Q!Xj|%wugsH$P~;N#>hkO7?%~btjK)=IT@Jl4T9D9`Sr zOedD>is7BZ4KAk{$`lZ z?5oTSsT-x%AFhwTRxY1?+>!1%)vBq;w^*+C)v1urmhO5`PXD-!pTza)7RLE3NN4-y zT4rVmp?=d+tm+X%xCw9w(9=p8JWI@>ys9m#2v))5uM)bWqduWOLc+&rCHZ)mFapE7-meAY3s zSGiti)c(^qw{S0jERW!3+^2n8maL2pIeys>Cvf&qO>sY+QShi zOLXS!#r5=-2E9+x7jIZtK;H%Of4gO*;+@?M^8ME*I*L-SloC6r)Hjp+gw+cv;H)J1 zhq!c7euU7bhNl+_C;bVJqTy-9!jJSX8eV+=9RvS4yvmS&4mThA&*9yM{d4#MuYV4Y z_x|Vb-+cZ#{G)Ht@Z$NS7gaR;WU>646jL<(XtD4!u|>nL77I6-R5ZMJ{+mxO8eX{k zseDVviEwhKA#O?az+y_#bc!FR&D4Jm|NiHH4lf`7&*8U=g*z6@hr9EN?q9qddMqd! zUOXS_FDe>dyd2UKiiW2aJHGX{qT$8MY4Dvdc^gp3T|8c5#+P^-3b|v79Zxs&pN|(- z)t-v9pgfD`$JrYH9G+I|pTlp|{^#&64T^>ruP;ioe-2M>Q8c``{pKx;h8Hi-dL2bL z<-sl6FYc$(t=9A6`R~}KXu5^%-agn~ISQfeTl?=DVy~Qq6b>)EKj~)?^#}*AQ-d}}JDjHtcezJQ~T+#4d#q9T= zW?ztZk%ipF&%@;BqUjVrk45n!oQ^pX?r)XHz0J@=Xj8+Bmq*scqT$8QXLYggZN<)K z=oPVlrqo9k3qSX&X!^y^v-{hk;l=a&Kv@TFKmB`V=umJT#qF$&urAxn6vXv}d)Gu= zC1L9q-R2!|$LcDYLh;T+w;A!iwT1j%M(MGyj4}f8CqZUI)okmklB!xkj;?8kaWmn zh_VaEgP1_9Aa;-*kN`+DWHuxTvJH{~xeUpIeoGc(eK-?jdAoC%+AZH<2kkY$x z9*{PWZjgQuUr0D41~M122C@Tk6mlMN9r6(J24b|QjA9D0fjC13Lq<9DEb`I>Lfd}iC2ek09@uIr4xwW0 z<<3!ozKRkWp6cTP8#XOkeAkfR`hh?@8=gDfmNWLqB|+R6z&@88x`c&#Vf>@l2TTXj@E|m8vA*)b8N-%UXk3z$pv6gmJxStx1kJZ~x9Bs2+@5rFOA)}G)N`0i{6YA^d z78*z?DxVZAA704cOUEfXP+j0#8U7_#MJYRB0db|a(kYxT^xY{eFrrhBK3$bRbzP%^ zA_MKmMEbdh_7C*+a|rMXSI+3GPWRA)aIDYJ+t16F3Q585ad|xuX)E<~9HYX5aDtJt zPT_aWqGro{L5bWWi`@N!!`!?g1K__*eo-dwfhh9`N|;KytRbyTp2`Xjg?WmRzS1bgj{H2;F=2j+zK)|`kYD6iS)L;H@(PcP3Zn+oU%8{- zAH~JlNh}KE};2oG-8*;^XHK8pTVcEX9|$O5j0y}7 z4GBh->4Q2L;va#OR`7cLt)P&iEsU36Jxa$h9818v@4 z%39i|lY<);aG(@>I=Pa2Cr`+A_ar>m=j%8JMMVTqM%qUWAC9gR4_FEvn^O2d zJvPOV&C@AbQGV6$?-dw{o~jQDJIGJ0$*)-Px`3w2Z4&G8uP9|@y{3crS5E7?qJJefnf^|;sNU z)?r7*?Kb@Rkm^oKYHcUwWD_SPthtjC5aFaGk9NZP3r>pXG$*{Bg79@tisj#MKXX#{ zzC^q{C#B|RCuOdSvtl{GSuq&wtjzVHyMAA`9uGAKw%5|4u81uv5$}RS)J+GD>X1@W zXhcqKJxEQ6KBPRPG{g`>qdX&sfh6h3Ulu}P zAf3x(2!+Q%$ewwS?;tCse>F((RO6#pn=ifxom;V8v|`Ee-Df42YRz<(_VGFTYCwL_c1F|p2#9vFZ9|11;a=7IQg z5x*p9CJplP3B}0b%jOk<&psF0ydu4d_=`nXf1NQNK-61CE~|$JQ4Y_ z6-!Y52`ebLteWkOa+Mu>|m8vPKUu8-9lEVM1 z@zr!xs{8+k|9%b>{ff%hzEZ_DUAd`8ZmK%|Z5o%jiKNX+-2UxD|8Gj5)Tvy!S9(SVq?_2S_&7NX>z~fqZ}{>zx%n zhylb1QWatfv4Ge>Y$1*iH;6mL12P!m3GsylK!PA)kVr^0BnA=-nGQ*SEQe71wUEt_ zU66y2laMsXO-L5xDI^>60b;ztxnQhs3bzHs3epN<3vq_tP|P9q7t|`ODSq6l2#g%>tkeN_fhJ%QSQ|_P>w#%t70?%~4`zZD zKu3_~j%MBD+hsQ;8T$1lY>E;V@8A3KutrjHr{6V$>_Nt z%}KjJPjh)Ukk5Ugr_-DcHUK@KHwOoU7N95C5cCBbfkB|y)N%708nR-0A;-}Llh;@o zG1l0zpnPI!0YR_CNUS-; z6%qtd_vwJJfskm(V#x21bjWK+CB(CV^ne6IV#9<@*)PTT->r{3V*TH5ZjS%n{stGZ zE-Zx3MYW=^Th&p$9*cUKfP&$3e?6QBJ`qdzO{gwtAfM5<#)1=(0F^u4k1>N#Vex%H zbyJ^Dt9~V^l%~LaQvZwl9slhV57I0*D$u(fsaW8)$f>NlV;3b>SKVd@Q)&sfp7vDj z*{#UE`q|m@wGUtFp)t!|`FmK?J=J;tg5*vIr;O~|xc#jBu#6b{ zgSTeie!JrRlkppSIjnv->i*eR-!+e$*vQe+xW>jeTa51?JNnM?$%gdT`A_b48@up= zli%4B^D3<0l2q&dFV5xtY8XXLoEkOiR2Nqd`^*gs+IMKZcgM3nt{WZ<>3D2NhnXwq z5Bxl2efDazRQGl9ZN2{dICJ34s||g{>7 zVHD0+Id}HlS+TumKXxp8<(Dw}_l6Dh?Qo~(ti>Uf@mrT8m#qA1-Seoi=g*fct)h8c z=h~e&AKUPpY1Z!!4BD#yN871SSj&zF?Z-a&D?xktZ+7_O$Marsl}4Z4l*yXA47%%j zG$mo-!&7Yj(LNob9M4rvS-O?o_TQh--s!ix+b<+A3-jam4({*J!fWPO=GAw=q4C@M z#8f^oj9q!&d{*g{`BSRBcVrf9-J|yF8b6+p+=PwWygKusZ;W%=t-f|XKb1af?ml|* z=Xn!5&U^Pmt+7509a^Mrwyi(oPb@dSc(3yDN{5@%Jq&7Z&$>T#y>5r&aZdJSb}xx) zd!yM8Eq^q>n5#V^ZWPr0(Ml|(0i5RfIY@m4ZCJqw`tY0 z&zAEe8_pS^w>I_m;c@S?j@MoJ_w<4N=hfKs@_5X!gJCzV^I=r7mn~CM#f`9aNSo9y?(9?ST^e&|*TG=>?RBW?l(ZNAxqr{`? zj*VM1vuSMRN6$R*_@rawzV6P}?HZd!;4X!)R}h|pbZk7vFQPHCD__>uDVhw!2VNOj1lT zOpcu>bIff~8}{PKVO^?*FQH8_JCI(g3I?6~aU-(IiVXAQb*&W!1k@9q^hds@GV z!6!F+y{6orJ!^8WxW0cJ$L;%{pT+I_cWZL{{&8`5yW98oxZ#I?*o%MpV-$EAzlb5d z$y~C8yiL9$rr{X@-W`AMg?48BrxZ6M1Hs)xk=#9<_?C_113dxOI&*<#M@Cc2`o(Q%|V_>QnVs zW!uClbl!EwxTeSS4o&>wJ{A-Qe#XlvGMdcCx;`M^lAlNmBg$~GwlT)-#$sc&u>tEl zYsAz3bRZpx^-Z8t>D}~xtgx7t(l_WX`e)ie+p&0NvZ+|X=KkxS%uxmS)<)6`nE4N<+$o?#!czqNm{ z`#bZo)6Y6@Ie&4!aGJYUx>c_B(!IIfaW7U6&}n*t&e0F(HOPWb^w+wD-_y7KOn*%< zA9A46@DfFakx|B1*z<98A}wW4iFd?4aY}qI23SL_d#%Tf}00 zoORA_=L4s$OWc+221M;XZ?K-Irv=wv7Q*GgokkCq!|&%C_)*?oEHamxCFZAATiHRr zF5i{)c6aAh=U8yf75E}EUS^Od@hIO^`T#vf!`U!4l4Y`)Y(94JIkuVIhdMrr+#Djd zih!A54mNYlb>@fWr{*~`+PcO{vr4VMS^corWpajEuAWepszxQ+H`x>HHTE|9bGx(+tU@U?M`qXa+kZS-DlnP?yGLOyUqQRd&vFF{mzZ>x_Y;I8QyfS z*n7h};a&9F=-$YRX?nTdtXukR{Hy&Devbc)zdLx{RhMe33t?m^nM4*K%i9^k7-BRT zJ!lm@&i=+bi3i2=qFi)0`Qp-0++*(D-g)nU{>TqBUecyvWFmQ#JZg-g%V=-5f+v`ZtwJkU zYyIVoaL9B;kdSO5+0@XCN=) zTX;wDlK4iB0C8(?!U9BI1yFpd`RD!j)u-WV4agvCIzam@JM*vv2T zGoqP!+`P?NC&Sb(wO1Xs>+K{b%$=e~`A_)4#v>m8zZJQTq>?OSgX}V^TnQW$*L$QZxX1bYSW}1`CEbL~EnQP`@ zPxH-%W&tw3&|GO2nZ?Nb^=7HL$t*L=&8_ARGufJMJ&&uBYJwW%OmOaWW?^?%I7Ln! zaOkhjwQe8erh}Y(-|g&m^O!fud&Ilo{o*y#t@P!(y}n9!*4=aub?3){f!P}hK25Zn-dnAJ1?Xw z=xcNbJwd;wtywhV%xB}-9c&4ElD)|)SRFgV+VGjEj@A4*?pdkURBNvFy!Dy2TD~Z^ zfSq1d=hPK8wddMT*jFJsHsX`!)Hz(Z}`oI@&My-}7t0_vif3-FouFLY&_QdMb$wM!t_B4*;E3kd@>OvH<-5 zlko}Vi0NeZ2$Vn%Zt@rTW?scV;(NtN^RQ`KIo31QLCXXRoRX`cG>V-rZln97*GXTa zXX*{QT6gwu^KlQd!m!Y@6WWtf5^GEd9owNvbTGY*wnaqVVMp0lY%TwuJ0e;1Gmn{9 zS^vWtVU4%)tU7Cy93$_L&&$u`WOa{v2WspS^_`-q`hE5pyRFm7xeifz$oU%h-NNnV zdhSi`?LdNu+&%8Kp5+bl(!BZJQg5yInz!BC6N=U`?=!E9P6v-~*M{%-6a43c`&JTm zNjs2UgafM!$P=W5yiUF&V~u-^g{b%%<8!FYX3%jj(Jk}s$sNsFkdGA|?tPib^p^QJZzOcSV{hkFU{RCzWlPzRxdAV#a z$Eo-2kKF!xkN&If<@bfwd&=MCAM-#e}@u&1F`YpYI-N%+AsvUR&r(AN)lc7al;T}+Sfw{swVxBb5 zm_M5>EX&HYW?2iYJ=QVnlogPzu&U0o8#r^I9EA+MUCxsa%W}D2c2u6aP2H}hsK?bS zP=1x_kUF6*x37er?q&D42VzH7+ZA@TebPRQ{K|4JLIYk644Uh{=I!#1dY^hxdN%am z!@5N8LYAMY|EuZg#dh2OZ*$hN?RQ#Ga?0 z&|m3({;mG)ey;zR|Fpl&KY$}N3w;*mWpaj`Cru=U-bBmj7&Zawbp`vCMRSKQw&uvw zs!??WemZtPyVhx`FVk)H6{!EKbr&#IwBH2|0ry?MKX!i%&XD8J_4ED5{bFSLPXB%X zsDHu_-E}Og89rN$&(vUVt|YOf2k?LpNj!1`83I(eg^Ys|m_}w|-{z5p$e9)7X=s?| z$wu-jd6Vn}OIDI9@*(+zd`{}o{hTKkNtn^vXlq0pu|_xSa$kdkY5GHP4l`~xMjM$% zmT{*s%ec?THy*~`KLH*39D0sTsIYCuE@MA1{V+KDr11qZ;yYx1fVQA*Xa^cYyU?EW zI`jpKYV-y}=?HXF`4`Ed|c(XGhox z)&f{IoX_L)p+dLw-O!L<@F-Djo> zK$eopMzQgh5k|K{d9P(}fCtXANZx~cJd@}0r+EqA$Pe*Typ4EHY=kbX6km%Q&1L3` z<`(k=IJCVr%(?^hw^6z zF1ntcrx)pEOrld*&8pbvY!(>d1E|+uM1<)DM#B4GbS@WR?A4YYJx67Q7?i9Dk zUFVj$+o7O8aL0R-y?eX@?>(>Hdrrq9Hw*pH-7%Azg`T(5+u-Pcqxg8JkmjO|aK!+T zDjr27wulPx9+bve@i);Sgr+&r$8D|d)--E2_UlP&FDie8{6)5d9v!GQsLiTcT>!#H zLhsMBKe0PI-JP}01}NT6?qYW<5OamM4p{lFcgpLf$NEc9AI}E&Dz#bYnOd94Nz%M+pZ7uCuhjGY^4--4VPhu&n0eTRLQeYZUa z%>RJ>nEivD4J~vWYI%m6hpxo*9B&vpkO#d*9@Un1^(bAe*XftRxN-iqzCb@Z2?$(_ z-NijdGn<7xibxWywv|SS@q)3@c-7cyw4qVJmoz#Sj>L4DgZeo}PtjkYR`S?u>}@zD zt&o>CPk-kC}flYs~XzmL=dZ+=;&JpnOS{qdWXWt+6-QuRGhF7u~b& z&u({b9@wZv_w|SPGyOnLv(P+by}8j0kRdg!?v)=`~_al_n=D8^C^gU0IJeAhoB->nJ<_>AP26qlC5>X{s)nFwtc~gal5+x z+`HYw=xn}plf4n%{fJa4b~;xt1!|X}bKj>2`8WGx{b~MJ@IXR$R4lx-@6CaQCVYwY zSkEEjV}nu~zS4SVx#RQ;8UZ|cnC)YS*jX0FFXtWkV*Vsw!`q6^qL-K`9v7?8&3z)Jco>Jsd_0#-J+(d995uJLUnFZ zr_}f0jW9a~PETL^Mtc}^ax$EWADkt~fDOpsN+`guy&uqm zhkKHiGz;C`_F=M|JVn;RS2<1E0F51B?JBwn=yMWRa~B*j6ACmRe%NbJ^M4T^o7Y;0 z)qcCie#_mBuD7|D>pkohd987tPUtUuJrFqZCe+1;`co~Tp6>7;0e`LYUx$XdgeCXF z4O>hegO1rkcA<){hZ{D`xXrj1zRqc2dOO+`IBp};u4Fx+-iNa^HVs^V1Z+Q==kR=f z9R0^nyoKl>s2Bz{KTZrW-!=cyV&n%Nqfm^bdf5L~0G%-fpF76d8#AfjU zI-y@gbM!%-&2H!r1v>k|=B?-gA2MGvx0_?-ba@uNhOKT?)3N7s)I(~iTBlxy9yth1 zs8Jj3a{IXbm7RtjS-A=D(gs1zYp4a>bGNQp=$Um-A}9A_7f;aB^k*7i?N~?FlO-~W zD@bBDhGIGz%3(UYo6Thla1F13ZFb-q4x&@K8j5NN>iRwYF<%6?VMi#sRlw95IHJ9* zpih+w2PNIgfVP@sWreV9hgAWb?Jb2&mAA?anF$0~Bwv-E%6i!VKk(ciJ=AcOrqWf0 z%2f9`UEO6~fgif-YGbp|^T|38i`+#XhYtLeBtReJ0ht=0tTW-Nr+%xV6e&`OTf##uSd-Z^Fxd*wog^WPA zF$>E5eW3Rz^hbI<6y!*LJL+{Ee;wU!j7Wqlaa25R9)yy-#=6!L))TmzpJWTwO-;hS zRHF{Bum{*f>{0N%K19dyiyi0mLTCPnv&X4%PC2dJD0i59i+i`f5QtmqzwIA{whGG#ie1KIc^{sHbu5LBO@@+9gI8eV9D%szArChL3g&9;<1=t@ zcCdB4im11v!f}5A~cWBn{s9yQGWJ+sH9? z8*S-3bTWLPr=gqQLq`$`lo}`20mVDvtdpQ9+E`tzu~7J};imd>fSfOP%WITU1Jn>` zj1#p^=!fT!Z)z+22#&^QQ8zshzyTs2}TZ*|0uHe_! zW9k~5X(u|pLv|fFaDlVa8Q{L)Z-I~8Jmj-p$NDnHEasrMz5#VR1eH4qwHtH-Ch&$3 zXI%^5VlHrbg}DaKAUaF=mOL+8sk3S?dg*}sj{6Z=8@5Q?z~s-i@%*QI(BII~=D z1#ecsfvMC7p(3l%UDoP4eM;Bs27LkhjV}-R5q_i}20W~~{XYp*XK`uO!d8n5{z7if_ zF;9XMnj(h+gVJRBZzz-nX3degVApxz)va=etdiBT20U@lLQ|* z#Tg3!D$PlEGT^;WLQjwlCo9*kh(NILOcAOpmha;Eb4aLOie;?C- zCxaXHxqr&yKw--eU)7LWQb+!0HJ^e~{k@`_(4U6W2pUO)x}X7blqMFASVkkqPD4&lLO$mrmlq;dfs&S?p6Bmz;$I&*ibkXRjiqr=#|bo%63VHhE?l%^nu0z! z6)HX*bI(ll!dWyMT0WQlz89&#@$f(AasgCFFrQJIbuB}_&|q}NBl#^n5QXoj)miK!b|W{z}5v|wbQB%DN`mZMP3 zF{tM_RCEGrnxLvB>N*LPor2m Date: Thu, 6 May 2021 15:58:21 +0300 Subject: [PATCH 06/44] remove contribs --- contrib/density/.gitignore | 2 - contrib/density/.gitmodules | 6 - contrib/density/.travis.yml | 272 --- contrib/density/CHANGELOG.md | 168 -- contrib/density/LICENSE.md | 28 - contrib/density/README.md | 261 --- contrib/density/appveyor.yml | 37 - contrib/density/benchmark/src/benchmark.c | 345 ---- contrib/density/benchmark/src/benchmark.h | 129 -- contrib/density/msvc/Density.sln | 21 - contrib/density/msvc/benchmark.vcxproj | 78 - .../density/msvc/benchmark.vcxproj.filters | 63 - contrib/density/msvc/density.vcxproj | 93 - contrib/density/msvc/density.vcxproj.filters | 135 -- contrib/density/src/algorithms/algorithms.c | 43 - contrib/density/src/algorithms/algorithms.h | 78 - .../src/algorithms/chameleon/chameleon.h | 70 - .../chameleon/core/chameleon_decode.c | 254 --- .../chameleon/core/chameleon_decode.h | 53 - .../chameleon/core/chameleon_encode.c | 179 -- .../chameleon/core/chameleon_encode.h | 53 - .../dictionary/chameleon_dictionary.h | 63 - .../density/src/algorithms/cheetah/cheetah.h | 73 - .../algorithms/cheetah/core/cheetah_decode.c | 266 --- .../algorithms/cheetah/core/cheetah_decode.h | 54 - .../algorithms/cheetah/core/cheetah_encode.c | 202 --- .../algorithms/cheetah/core/cheetah_encode.h | 54 - .../cheetah/dictionary/cheetah_dictionary.h | 70 - contrib/density/src/algorithms/dictionaries.c | 48 - contrib/density/src/algorithms/dictionaries.h | 45 - .../src/algorithms/lion/core/lion_decode.c | 327 ---- .../src/algorithms/lion/core/lion_decode.h | 54 - .../src/algorithms/lion/core/lion_encode.c | 298 ---- .../src/algorithms/lion/core/lion_encode.h | 54 - .../lion/dictionary/lion_dictionary.h | 73 - .../algorithms/lion/forms/lion_form_model.c | 153 -- .../algorithms/lion/forms/lion_form_model.h | 96 - contrib/density/src/algorithms/lion/lion.h | 94 - contrib/density/src/buffers/buffer.c | 220 --- contrib/density/src/buffers/buffer.h | 58 - contrib/density/src/density_api.h | 220 --- contrib/density/src/globals.c | 47 - contrib/density/src/globals.h | 232 --- contrib/density/src/structure/header.c | 57 - contrib/density/src/structure/header.h | 58 - contrib/lizard/.gitattributes | 21 - contrib/lizard/.gitignore | 37 - contrib/lizard/.travis.yml | 268 --- contrib/lizard/LICENSE | 15 - contrib/lizard/NEWS | 41 - contrib/lizard/README.md | 95 - contrib/lizard/appveyor.yml | 143 -- contrib/lizard/contrib/djgpp/LICENSE | 24 - contrib/lizard/contrib/djgpp/README.MD | 21 - contrib/lizard/doc/lizard_Block_format.md | 181 -- contrib/lizard/doc/lizard_Frame_format.md | 312 ---- contrib/lizard/examples/.gitignore | 8 - .../lizard/examples/HCStreaming_ringBuffer.c | 241 --- contrib/lizard/examples/README.md | 8 - .../examples/blockStreaming_doubleBuffer.c | 203 --- .../examples/blockStreaming_doubleBuffer.md | 100 -- .../examples/blockStreaming_lineByLine.c | 210 --- .../examples/blockStreaming_lineByLine.md | 122 -- .../examples/blockStreaming_ringBuffer.c | 202 --- contrib/lizard/examples/compress_functions.c | 303 ---- contrib/lizard/examples/frameCompress.c | 169 -- contrib/lizard/examples/printVersion.c | 13 - contrib/lizard/examples/simple_buffer.c | 91 - .../lizard/examples/streaming_api_basics.md | 87 - contrib/lizard/lib/.gitignore | 3 - contrib/lizard/lib/LICENSE | 25 - contrib/lizard/lib/README.md | 76 - contrib/lizard/lib/dll/liblizard.def | 19 - contrib/lizard/lib/entropy/README.md | 38 - contrib/lizard/lib/entropy/bitstream.h | 414 ----- contrib/lizard/lib/entropy/entropy_common.c | 231 --- contrib/lizard/lib/entropy/error_private.h | 115 -- contrib/lizard/lib/entropy/error_public.h | 64 - contrib/lizard/lib/entropy/fse.h | 694 -------- contrib/lizard/lib/entropy/fse_compress.c | 848 --------- contrib/lizard/lib/entropy/fse_decompress.c | 329 ---- contrib/lizard/lib/entropy/huf.h | 250 --- contrib/lizard/lib/entropy/huf_compress.c | 612 ------- contrib/lizard/lib/entropy/huf_decompress.c | 885 ---------- contrib/lizard/lib/entropy/mem.h | 372 ---- contrib/lizard/lib/liblizard.pc.in | 15 - contrib/lizard/lib/lizard_common.h | 504 ------ contrib/lizard/lib/lizard_compress.c | 630 ------- contrib/lizard/lib/lizard_compress.h | 208 --- contrib/lizard/lib/lizard_compress_liz.h | 301 ---- contrib/lizard/lib/lizard_compress_lz4.h | 162 -- contrib/lizard/lib/lizard_decompress.c | 372 ---- contrib/lizard/lib/lizard_decompress.h | 152 -- contrib/lizard/lib/lizard_decompress_liz.h | 220 --- contrib/lizard/lib/lizard_decompress_lz4.h | 163 -- contrib/lizard/lib/lizard_frame.c | 1362 -------------- contrib/lizard/lib/lizard_frame.h | 303 ---- contrib/lizard/lib/lizard_frame_static.h | 81 - contrib/lizard/lib/lizard_parser_fast.h | 196 --- contrib/lizard/lib/lizard_parser_fastbig.h | 175 -- contrib/lizard/lib/lizard_parser_fastsmall.h | 189 -- contrib/lizard/lib/lizard_parser_hashchain.h | 369 ---- .../lizard/lib/lizard_parser_lowestprice.h | 376 ---- contrib/lizard/lib/lizard_parser_nochain.h | 318 ---- contrib/lizard/lib/lizard_parser_optimal.h | 679 ------- contrib/lizard/lib/lizard_parser_pricefast.h | 250 --- contrib/lizard/lib/xxhash/xxhash.c | 888 ---------- contrib/lizard/lib/xxhash/xxhash.h | 293 --- contrib/lizard/programs/.gitignore | 15 - contrib/lizard/programs/COPYING | 339 ---- contrib/lizard/programs/README.md | 73 - contrib/lizard/programs/bench.c | 502 ------ contrib/lizard/programs/bench.h | 37 - contrib/lizard/programs/datagen.c | 188 -- contrib/lizard/programs/datagen.h | 39 - contrib/lizard/programs/lizard.1 | 229 --- contrib/lizard/programs/lizardcli.c | 581 ------ contrib/lizard/programs/lizardio.c | 896 ---------- contrib/lizard/programs/lizardio.h | 101 -- contrib/lizard/programs/platform.h | 145 -- contrib/lizard/programs/util.h | 497 ------ contrib/lizard/tests/.gitignore | 13 - contrib/lizard/tests/COPYING | 339 ---- contrib/lizard/tests/README.md | 15 - contrib/lizard/tests/datagencli.c | 192 -- contrib/lizard/tests/frametest.c | 866 --------- contrib/lizard/tests/fullbench.c | 732 -------- contrib/lizard/tests/fuzzer.c | 1086 ------------ contrib/lizard/visual/.gitignore | 11 - contrib/lizard/visual/README.md | 53 - .../visual/VS2010/datagen/datagen.vcxproj | 165 -- .../visual/VS2010/frametest/frametest.vcxproj | 177 -- .../visual/VS2010/fullbench/fullbench.vcxproj | 175 -- .../visual/VS2010/fuzzer/fuzzer.vcxproj | 174 -- .../VS2010/liblizard-dll/liblizard-dll.rc | 51 - .../liblizard-dll/liblizard-dll.vcxproj | 179 -- .../visual/VS2010/liblizard/liblizard.vcxproj | 176 -- contrib/lizard/visual/VS2010/lizard.sln | 86 - contrib/lizard/visual/VS2010/lizard/lizard.rc | 51 - .../visual/VS2010/lizard/lizard.vcxproj | 190 -- contrib/lzsse/.gitignore | 4 - contrib/lzsse/.travis.yml | 185 -- contrib/lzsse/LICENSE | 23 - contrib/lzsse/README.md | 15 - contrib/lzsse/example/main.cpp | 389 ---- contrib/lzsse/lzsse2/lzsse2.cpp | 1080 ------------ contrib/lzsse/lzsse2/lzsse2.h | 90 - contrib/lzsse/lzsse2/lzsse2_platform.h | 73 - contrib/lzsse/lzsse4/lzsse4.cpp | 1499 ---------------- contrib/lzsse/lzsse4/lzsse4.h | 117 -- contrib/lzsse/lzsse4/lzsse4_platform.h | 73 - contrib/lzsse/lzsse8/lzsse8.cpp | 1568 ----------------- contrib/lzsse/lzsse8/lzsse8.h | 117 -- contrib/lzsse/lzsse8/lzsse8_platform.h | 73 - contrib/lzsse/premake4.exe | Bin 518144 -> 0 bytes contrib/lzsse/premake4.lua | 26 - 156 files changed, 35085 deletions(-) delete mode 100644 contrib/density/.gitignore delete mode 100644 contrib/density/.gitmodules delete mode 100644 contrib/density/.travis.yml delete mode 100644 contrib/density/CHANGELOG.md delete mode 100644 contrib/density/LICENSE.md delete mode 100644 contrib/density/README.md delete mode 100644 contrib/density/appveyor.yml delete mode 100644 contrib/density/benchmark/src/benchmark.c delete mode 100644 contrib/density/benchmark/src/benchmark.h delete mode 100644 contrib/density/msvc/Density.sln delete mode 100644 contrib/density/msvc/benchmark.vcxproj delete mode 100644 contrib/density/msvc/benchmark.vcxproj.filters delete mode 100644 contrib/density/msvc/density.vcxproj delete mode 100644 contrib/density/msvc/density.vcxproj.filters delete mode 100644 contrib/density/src/algorithms/algorithms.c delete mode 100644 contrib/density/src/algorithms/algorithms.h delete mode 100644 contrib/density/src/algorithms/chameleon/chameleon.h delete mode 100644 contrib/density/src/algorithms/chameleon/core/chameleon_decode.c delete mode 100644 contrib/density/src/algorithms/chameleon/core/chameleon_decode.h delete mode 100644 contrib/density/src/algorithms/chameleon/core/chameleon_encode.c delete mode 100644 contrib/density/src/algorithms/chameleon/core/chameleon_encode.h delete mode 100644 contrib/density/src/algorithms/chameleon/dictionary/chameleon_dictionary.h delete mode 100644 contrib/density/src/algorithms/cheetah/cheetah.h delete mode 100644 contrib/density/src/algorithms/cheetah/core/cheetah_decode.c delete mode 100644 contrib/density/src/algorithms/cheetah/core/cheetah_decode.h delete mode 100644 contrib/density/src/algorithms/cheetah/core/cheetah_encode.c delete mode 100644 contrib/density/src/algorithms/cheetah/core/cheetah_encode.h delete mode 100644 contrib/density/src/algorithms/cheetah/dictionary/cheetah_dictionary.h delete mode 100644 contrib/density/src/algorithms/dictionaries.c delete mode 100644 contrib/density/src/algorithms/dictionaries.h delete mode 100644 contrib/density/src/algorithms/lion/core/lion_decode.c delete mode 100644 contrib/density/src/algorithms/lion/core/lion_decode.h delete mode 100644 contrib/density/src/algorithms/lion/core/lion_encode.c delete mode 100644 contrib/density/src/algorithms/lion/core/lion_encode.h delete mode 100644 contrib/density/src/algorithms/lion/dictionary/lion_dictionary.h delete mode 100644 contrib/density/src/algorithms/lion/forms/lion_form_model.c delete mode 100644 contrib/density/src/algorithms/lion/forms/lion_form_model.h delete mode 100644 contrib/density/src/algorithms/lion/lion.h delete mode 100644 contrib/density/src/buffers/buffer.c delete mode 100644 contrib/density/src/buffers/buffer.h delete mode 100644 contrib/density/src/density_api.h delete mode 100644 contrib/density/src/globals.c delete mode 100644 contrib/density/src/globals.h delete mode 100644 contrib/density/src/structure/header.c delete mode 100644 contrib/density/src/structure/header.h delete mode 100644 contrib/lizard/.gitattributes delete mode 100644 contrib/lizard/.gitignore delete mode 100644 contrib/lizard/.travis.yml delete mode 100644 contrib/lizard/LICENSE delete mode 100644 contrib/lizard/NEWS delete mode 100644 contrib/lizard/README.md delete mode 100644 contrib/lizard/appveyor.yml delete mode 100644 contrib/lizard/contrib/djgpp/LICENSE delete mode 100644 contrib/lizard/contrib/djgpp/README.MD delete mode 100644 contrib/lizard/doc/lizard_Block_format.md delete mode 100644 contrib/lizard/doc/lizard_Frame_format.md delete mode 100644 contrib/lizard/examples/.gitignore delete mode 100644 contrib/lizard/examples/HCStreaming_ringBuffer.c delete mode 100644 contrib/lizard/examples/README.md delete mode 100644 contrib/lizard/examples/blockStreaming_doubleBuffer.c delete mode 100644 contrib/lizard/examples/blockStreaming_doubleBuffer.md delete mode 100644 contrib/lizard/examples/blockStreaming_lineByLine.c delete mode 100644 contrib/lizard/examples/blockStreaming_lineByLine.md delete mode 100644 contrib/lizard/examples/blockStreaming_ringBuffer.c delete mode 100644 contrib/lizard/examples/compress_functions.c delete mode 100644 contrib/lizard/examples/frameCompress.c delete mode 100644 contrib/lizard/examples/printVersion.c delete mode 100644 contrib/lizard/examples/simple_buffer.c delete mode 100644 contrib/lizard/examples/streaming_api_basics.md delete mode 100644 contrib/lizard/lib/.gitignore delete mode 100644 contrib/lizard/lib/LICENSE delete mode 100644 contrib/lizard/lib/README.md delete mode 100644 contrib/lizard/lib/dll/liblizard.def delete mode 100644 contrib/lizard/lib/entropy/README.md delete mode 100644 contrib/lizard/lib/entropy/bitstream.h delete mode 100644 contrib/lizard/lib/entropy/entropy_common.c delete mode 100644 contrib/lizard/lib/entropy/error_private.h delete mode 100644 contrib/lizard/lib/entropy/error_public.h delete mode 100644 contrib/lizard/lib/entropy/fse.h delete mode 100644 contrib/lizard/lib/entropy/fse_compress.c delete mode 100644 contrib/lizard/lib/entropy/fse_decompress.c delete mode 100644 contrib/lizard/lib/entropy/huf.h delete mode 100644 contrib/lizard/lib/entropy/huf_compress.c delete mode 100644 contrib/lizard/lib/entropy/huf_decompress.c delete mode 100644 contrib/lizard/lib/entropy/mem.h delete mode 100644 contrib/lizard/lib/liblizard.pc.in delete mode 100644 contrib/lizard/lib/lizard_common.h delete mode 100644 contrib/lizard/lib/lizard_compress.c delete mode 100644 contrib/lizard/lib/lizard_compress.h delete mode 100644 contrib/lizard/lib/lizard_compress_liz.h delete mode 100644 contrib/lizard/lib/lizard_compress_lz4.h delete mode 100644 contrib/lizard/lib/lizard_decompress.c delete mode 100644 contrib/lizard/lib/lizard_decompress.h delete mode 100644 contrib/lizard/lib/lizard_decompress_liz.h delete mode 100644 contrib/lizard/lib/lizard_decompress_lz4.h delete mode 100644 contrib/lizard/lib/lizard_frame.c delete mode 100644 contrib/lizard/lib/lizard_frame.h delete mode 100644 contrib/lizard/lib/lizard_frame_static.h delete mode 100644 contrib/lizard/lib/lizard_parser_fast.h delete mode 100644 contrib/lizard/lib/lizard_parser_fastbig.h delete mode 100644 contrib/lizard/lib/lizard_parser_fastsmall.h delete mode 100644 contrib/lizard/lib/lizard_parser_hashchain.h delete mode 100644 contrib/lizard/lib/lizard_parser_lowestprice.h delete mode 100644 contrib/lizard/lib/lizard_parser_nochain.h delete mode 100644 contrib/lizard/lib/lizard_parser_optimal.h delete mode 100644 contrib/lizard/lib/lizard_parser_pricefast.h delete mode 100644 contrib/lizard/lib/xxhash/xxhash.c delete mode 100644 contrib/lizard/lib/xxhash/xxhash.h delete mode 100644 contrib/lizard/programs/.gitignore delete mode 100644 contrib/lizard/programs/COPYING delete mode 100644 contrib/lizard/programs/README.md delete mode 100644 contrib/lizard/programs/bench.c delete mode 100644 contrib/lizard/programs/bench.h delete mode 100644 contrib/lizard/programs/datagen.c delete mode 100644 contrib/lizard/programs/datagen.h delete mode 100644 contrib/lizard/programs/lizard.1 delete mode 100644 contrib/lizard/programs/lizardcli.c delete mode 100644 contrib/lizard/programs/lizardio.c delete mode 100644 contrib/lizard/programs/lizardio.h delete mode 100644 contrib/lizard/programs/platform.h delete mode 100644 contrib/lizard/programs/util.h delete mode 100644 contrib/lizard/tests/.gitignore delete mode 100644 contrib/lizard/tests/COPYING delete mode 100644 contrib/lizard/tests/README.md delete mode 100644 contrib/lizard/tests/datagencli.c delete mode 100644 contrib/lizard/tests/frametest.c delete mode 100644 contrib/lizard/tests/fullbench.c delete mode 100644 contrib/lizard/tests/fuzzer.c delete mode 100644 contrib/lizard/visual/.gitignore delete mode 100644 contrib/lizard/visual/README.md delete mode 100644 contrib/lizard/visual/VS2010/datagen/datagen.vcxproj delete mode 100644 contrib/lizard/visual/VS2010/frametest/frametest.vcxproj delete mode 100644 contrib/lizard/visual/VS2010/fullbench/fullbench.vcxproj delete mode 100644 contrib/lizard/visual/VS2010/fuzzer/fuzzer.vcxproj delete mode 100644 contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.rc delete mode 100644 contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.vcxproj delete mode 100644 contrib/lizard/visual/VS2010/liblizard/liblizard.vcxproj delete mode 100644 contrib/lizard/visual/VS2010/lizard.sln delete mode 100644 contrib/lizard/visual/VS2010/lizard/lizard.rc delete mode 100644 contrib/lizard/visual/VS2010/lizard/lizard.vcxproj delete mode 100644 contrib/lzsse/.gitignore delete mode 100644 contrib/lzsse/.travis.yml delete mode 100644 contrib/lzsse/LICENSE delete mode 100644 contrib/lzsse/README.md delete mode 100644 contrib/lzsse/example/main.cpp delete mode 100644 contrib/lzsse/lzsse2/lzsse2.cpp delete mode 100644 contrib/lzsse/lzsse2/lzsse2.h delete mode 100644 contrib/lzsse/lzsse2/lzsse2_platform.h delete mode 100644 contrib/lzsse/lzsse4/lzsse4.cpp delete mode 100644 contrib/lzsse/lzsse4/lzsse4.h delete mode 100644 contrib/lzsse/lzsse4/lzsse4_platform.h delete mode 100644 contrib/lzsse/lzsse8/lzsse8.cpp delete mode 100644 contrib/lzsse/lzsse8/lzsse8.h delete mode 100644 contrib/lzsse/lzsse8/lzsse8_platform.h delete mode 100644 contrib/lzsse/premake4.exe delete mode 100644 contrib/lzsse/premake4.lua diff --git a/contrib/density/.gitignore b/contrib/density/.gitignore deleted file mode 100644 index 5192545dd3f..00000000000 --- a/contrib/density/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -build/ -benchmark/build diff --git a/contrib/density/.gitmodules b/contrib/density/.gitmodules deleted file mode 100644 index 91eee619841..00000000000 --- a/contrib/density/.gitmodules +++ /dev/null @@ -1,6 +0,0 @@ -[submodule "benchmark/libs/cputime"] - path = benchmark/libs/cputime - url = https://github.com/k0dai/cputime.git -[submodule "benchmark/libs/spookyhash"] - path = benchmark/libs/spookyhash - url = https://github.com/k0dai/spookyhash.git diff --git a/contrib/density/.travis.yml b/contrib/density/.travis.yml deleted file mode 100644 index 62e7655a4cb..00000000000 --- a/contrib/density/.travis.yml +++ /dev/null @@ -1,272 +0,0 @@ -language: c -matrix: - fast_finish: true - include: - - os: osx - env: - - C_COMPILER=clang - - - os: osx - env: - - C_COMPILER=gcc - - - dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-arm - - qemu-user-static - - gcc-aarch64-linux-gnu - - g++-aarch64-linux-gnu - env: - - C_COMPILER=aarch64-linux-gnu-gcc MAKE_OPTIONS="AR=gcc-ar NATIVE=off" QEMU_INTERPRETER=qemu-aarch64-static PRE_SCRIPT="export QEMU_LD_PREFIX=/usr/aarch64-linux-gnu/" - - - dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-arm - - qemu-user-static - - gcc-arm-linux-gnueabi - - g++-arm-linux-gnueabi - env: - - C_COMPILER=arm-linux-gnueabi-gcc MAKE_OPTIONS="AR=gcc-ar NATIVE=off" QEMU_INTERPRETER=qemu-arm-static PRE_SCRIPT="export QEMU_LD_PREFIX=/usr/arm-linux-gnueabi/" - - - dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-ppc - - qemu-user-static - - gcc-powerpc-linux-gnu - - g++-powerpc-linux-gnu - env: - - C_COMPILER=powerpc-linux-gnu-gcc MAKE_OPTIONS="AR=gcc-ar ARCH=64" QEMU_INTERPRETER="qemu-ppc64-static" PRE_SCRIPT="export QEMU_LD_PREFIX=/usr/powerpc-linux-gnu/" - - - dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-ppc - - qemu-user-static - - gcc-powerpc-linux-gnu - - g++-powerpc-linux-gnu - env: - - C_COMPILER=powerpc-linux-gnu-gcc MAKE_OPTIONS="AR=gcc-ar NATIVE=off" QEMU_INTERPRETER="qemu-ppc-static" PRE_SCRIPT="export QEMU_LD_PREFIX=/usr/powerpc-linux-gnu/" - - - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-4.8 - env: - - C_COMPILER=gcc-4.8 MAKE_OPTIONS="AR=gcc-ar" - - - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-4.8 - - gcc-4.8-multilib - env: - - C_COMPILER=gcc-4.8 MAKE_OPTIONS="AR=gcc-ar ARCH=32" - - - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-4.9 - env: - - C_COMPILER=gcc-4.9 MAKE_OPTIONS="AR=gcc-ar" - - - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-4.9 - - gcc-4.9-multilib - env: - - C_COMPILER=gcc-4.9 MAKE_OPTIONS="AR=gcc-ar ARCH=32" - - - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-5 - env: - - C_COMPILER=gcc-5 MAKE_OPTIONS="AR=gcc-ar-5" - - - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-5 - - gcc-5-multilib - env: - - C_COMPILER=gcc-5 MAKE_OPTIONS="AR=gcc-ar-5 ARCH=32" - -# - dist: trusty -# sudo: required -# addons: -# apt: -# sources: -# - ubuntu-toolchain-r-test -# packages: -# - gcc-6 -# env: -# - C_COMPILER=gcc-6 MAKE_OPTIONS="AR=gcc-ar-6" - - - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-6 - - gcc-6-multilib - env: - - C_COMPILER=gcc-6 MAKE_OPTIONS="AR=gcc-ar-6 ARCH=32" - -# - dist: trusty -# sudo: required -# addons: -# apt: -# sources: -# - ubuntu-toolchain-r-test -# packages: -# - gcc-7 -# env: -# - C_COMPILER=gcc-7 MAKE_OPTIONS="AR=gcc-ar-7" - - - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-7 - - gcc-7-multilib - env: - - C_COMPILER=gcc-7 MAKE_OPTIONS="AR=gcc-ar-7 ARCH=32" - - - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-trusty-3.5 - packages: - - clang-3.5 - - llvm-3.5-dev - env: - - C_COMPILER=clang-3.5 MAKE_OPTIONS="AR=llvm-ar-3.5" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" - - - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-trusty-3.6 - packages: - - clang-3.6 - - llvm-3.6-dev - env: - - C_COMPILER=clang-3.6 MAKE_OPTIONS="AR=llvm-ar-3.6" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" - -# - dist: precise -# sudo: required -# addons: -# apt: -# sources: -# - ubuntu-toolchain-r-test -# - llvm-toolchain-precise-3.7 -# packages: -# - clang-3.7 -# - llvm-3.7-dev -# env: -# - C_COMPILER=clang-3.7 MAKE_OPTIONS="AR=llvm-ar-3.7" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" - -# - dist: precise -# sudo: required -# addons: -# apt: -# sources: -# - ubuntu-toolchain-r-test -# - llvm-toolchain-precise-3.8 -# packages: -# - clang-3.8 -# - llvm-3.8-dev -# env: -# - C_COMPILER=clang-3.8 MAKE_OPTIONS="AR=llvm-ar-3.8" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" - - - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-trusty-3.9 - packages: - - clang-3.9 - - llvm-3.9-dev - env: - - C_COMPILER=clang-3.9 MAKE_OPTIONS="AR=llvm-ar-3.9" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" - - - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-trusty-4.0 - packages: - - clang-4.0 - - llvm-4.0-dev - env: - - C_COMPILER=clang-4.0 MAKE_OPTIONS="AR=llvm-ar-4.0" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" - -# - dist: trusty -# sudo: required -# addons: -# apt: -# sources: -# - ubuntu-toolchain-r-test -# - llvm-toolchain-trusty-5.0 -# packages: -# - clang-5.0 -# - llvm-5.0-dev -# env: -# - C_COMPILER=clang-5.0 MAKE_OPTIONS="AR=llvm-ar-5.0" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld" - -script: - - ${PRE_SCRIPT} - - $C_COMPILER -v - - make CC=$C_COMPILER $MAKE_OPTIONS - - file ./build/* - - $QEMU_INTERPRETER ./build/benchmark -h -f diff --git a/contrib/density/CHANGELOG.md b/contrib/density/CHANGELOG.md deleted file mode 100644 index 6d5535bcc7c..00000000000 --- a/contrib/density/CHANGELOG.md +++ /dev/null @@ -1,168 +0,0 @@ -0.14.2 ------- -*February 12, 2018* -* Improved chameleon decode speed -* Added data hash checks and display option in benchmark -* Now using makefiles as build system -* Big endian support correctly implemented and tested -* Improved continuous integration tests - -0.14.1 ------- -*January 20, 2018* -* Added MSVC support -* Added continuous integration on travis and appveyor -* Premake script improvement -* Various codebase improvements - -0.14.0 ------- -*January 16, 2018* -* First stable version of DENSITY -* Complete project reorganization and API rewrite -* Many stability fixes and improvements -* Fast revert to conditional copy for incompressible input -* Custom dictionaries in API -* Improvements in compression ratio and speed - -0.12.5 beta ------------ -*June 20, 2015* - -* Added conditional main footer read/write -* Improved teleport staging buffer management -* Regression - a minimum buffer output size has to be ensured to avoid signature loss -* Modified the minimum lookahead and the resulting minimum buffer size in the API -* Lion : corrected a signature interception problem due to an increase in process unit size -* Lion : corrected chunk count conditions for new block / mode marker detection -* Lion : modified end of stream marker conditions -* Stability fixes and improvements - -0.12.4 beta ------------ -*May 25, 2015* - -* Removed remaining undefined behavior potential occurences -* Implemented parallelizable decompressible output block header reads/writes (disabled by default) - -0.12.3 beta ------------ -*May 20, 2015* - -* New lion algorithm, faster and more efficient -* Compiler specific optimizations -* Switched to premake 5 to benefit from link time optimizations -* Various fixes and improvements - -0.12.2 beta ------------ -*May 4, 2015* - -* Added an integrated in-memory benchmark -* Better Windows compatibility -* Fixed misaligned load/stores -* Switched to the premake build system -* Performance optimizations (pointers, branches, loops ...) -* Various fixes and improvements - -0.12.1 beta ------------ -*April 3, 2015* - -* Better unrolling readability and efficiency -* Improved read speed of dictionary/predictions entries -* Implemented case generators in cheetah to speed up decoding by using less branches -* Added signatures interception in lion to cancel the need for large output buffers -* Improved lion decode speed with specific form data access and use of ctz in form read -* Enabled decompression to exact-sized buffer for all algorithms -* Various fixes and improvements - -0.12.0 beta ------------ -*March 24, 2015* - -* Added new lion kernel -* Renamed kernel mandala to cheetah -* Kernel chameleon and cheetah improvements in encoding/decoding speeds -* Generic function macros to avoid code rewrite -* Improved memory teleport IO flexibility and speed, bytes issued by memory teleport can now be partially read -* Various fixes and improvements - -0.11.3 beta ------------ -*February 5, 2015* - -* Added integrity check system -* Corrected pointer usage and update on footer read/writes -* Now freeing kernel state memory only when compression mode is not copy -* Updated Makefiles -* Improved memory teleport -* Fixed sequencing problem after kernels request a new block - -0.11.2 beta ------------ -*February 3, 2015* - -* Added an algorithms overview in README -* Removed ssc references -* Now initializing last hash to zero on mandala kernel inits -* Reimplemented the buffer API -* Various corrections and improvements - -0.11.1 beta ------------ -*January 19, 2015* - -* Added a sharc benchmark in README -* Stateless memory teleport -* Improved event management and dispatching -* Improved compression/decompression finishes -* Improved streams API -* Various bug fixes, robustness improvements - -0.10.2 beta ------------ -*January 7, 2015* - -* Improved organization of compile-time switches and run-time options in the API -* Removed method density_stream_decompress_utilities_get_header from the API, header info is now returned in the density_stream_decompress_init function -* Corrected readme to reflect API changes - -0.10.1 beta ------------ -*January 5, 2015* - -* Re-added mandala kernel -* Corrected available bytes adjustment problem -* Added missing restrict keywords -* Cleaned unnecessary defines - -0.10.0 beta ------------ -*January 2, 2015* - -* Complete stream API redesign to greatly improve flexibility -* Only one supported algorithm for now : Chameleon - -0.9.12 beta ------------ -*December 2, 2013* - -* Mandala kernel addition, replacing dual pass chameleon -* Simplified, faster hash function -* Fixed memory freeing issue during main encoding/decoding finish -* Implemented no footer encode output type -* Namespace migration, kernel structure reorganization -* Corrected copy mode problem -* Implemented efficiency checks and mode reversions -* Corrected lack of main header parameters retrieval -* Fixed stream not being properly ended when mode reversion occurred -* Updated metadata computations - -0.9.11 beta ------------ -*November 2, 2013* - -* First beta release of DENSITY, including all the compression code from SHARC in a standalone, BSD licensed library -* Added copy mode (useful for enhancing data security via the density block checksums for example) -* Makefile produces static and dynamic libraries diff --git a/contrib/density/LICENSE.md b/contrib/density/LICENSE.md deleted file mode 100644 index c2b699c9b83..00000000000 --- a/contrib/density/LICENSE.md +++ /dev/null @@ -1,28 +0,0 @@ -Copyright (c) 2013, Guillaume Voirin - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/density/README.md b/contrib/density/README.md deleted file mode 100644 index 6c660dea1f5..00000000000 --- a/contrib/density/README.md +++ /dev/null @@ -1,261 +0,0 @@ -DENSITY -======== -Superfast compression library - -DENSITY is a free C99, open-source, BSD licensed compression library. - -It is focused on high-speed compression, at the best ratio possible. **All three** of DENSITY's algorithms are currently at the **pareto frontier** of compression speed vs ratio (cf. [here](https://github.com/inikep/lzbench/blob/master/lzbench18_sorted.md) for an independent benchmark). - -DENSITY features a simple API to enable quick integration in any project. - -Branch|Linux & MacOS|Windows ---- | --- | --- -master|[![Build Status](https://travis-ci.org/k0dai/density.svg?branch=master)](https://travis-ci.org/k0dai/density)|[![Build status](https://ci.appveyor.com/api/projects/status/rf7x3x829il72cii/branch/master?svg=true)](https://ci.appveyor.com/project/gpnuma/density/branch/master) -dev|[![Build Status](https://travis-ci.org/k0dai/density.svg?branch=dev)](https://travis-ci.org/k0dai/density)|[![Build status](https://ci.appveyor.com/api/projects/status/rf7x3x829il72cii/branch/dev?svg=true)](https://ci.appveyor.com/project/gpnuma/density/branch/dev) - -Why is it so fast ? -------------------- - -One of the biggest assets of DENSITY is that its work unit is **not a byte** like other libraries, but **a group of 4 bytes**. - -When other libraries consume one byte of data and then apply an algorithmic processing to it, DENSITY consumes 4 bytes and then applies its algorithmic processing. - -That's why DENSITY's algorithms were designed from scratch. They have to alleviate for 4-byte work units and still provide interesting compression ratios. - -**Speed pedigree traits** - -* 4-byte work units -* heavy use of registers as opposed to memory for processing -* avoidance of or use of minimal branching when possible -* use of low memory data structures to favor processor cache Lx accesses -* library wide inlining -* specific unrollings -* prefetching and branching hints -* restricted pointers to maximize compiler optimizations - -A "blowup protection" is provided, dramatically increasing the processing speed of incompressible input data. Also, the output, compressed data size will **never exceed** the original uncompressed data size by more than 1% in case of incompressible, reasonably-sized inputs. - -Benchmarks ----------- - -**Quick benchmark** - -DENSITY features an **integrated in-memory benchmark**. After building the project (see [build](#build)), a *benchmark* executable will be present in the build directory. If run without arguments, usage help will be displayed. - -File used : enwik8 (100 MB) - -Platform : MacBook Pro, MacOS 10.13.3, 2.3 GHz Intel Core i7, 8Gb 1600 MHz DDR, SSD, compiling with Clang/LLVM 9.0.0 - -Timing : using the *time* function, and taking the best *user* output after multiple runs. In the case of density, the in-memory integrated benchmark's best value (which uses the same usermode CPU timing) is used. - -Library|Algorithm|Compress|Decompress|Size|Ratio|Round trip ----|---|---|---|---|---|--- -**density** 0.14.2|Chameleon|0.092s (1085 MB/s)|0.059s (1684 MB/s)|61 524 084|61,52%|0.151s -lz4 r129|-1|0.468s (214 MB/s)|0.115s (870 MB/s)|57 285 990|57,29%|0.583s -lzo 2.08|-1|0.367s (272 MB/s)|0.309s (324 MB/s)|56 709 096|56,71%|0.676s -**density** 0.14.2|Cheetah|0.170s (587 MB/s)|0.126s (796 MB/s)|53 156 668|53,16%|0.296s -**density** 0.14.2|Lion|0.303s (330 MB/s)|0.288s (347 MB/s)|47 817 692|47,82%|0.591s -lz4 r129|-3|1.685s (59 MB/s)|0.118s (847 MB/s)|44 539 940|44,54%|1.803s -lzo 2.08|-7|9.562s (10 MB/s)|0.319s (313 MB/s)|41 720 721|41,72%|9.881s - -**Other benchmarks** - -Here are a few other benchmarks featuring DENSITY (non exhaustive list) : - -* [**squash**](https://github.com/quixdb/squash) is an abstraction layer for compression algorithms, and has an extremely exhaustive set of benchmark results, including density's, [available here](https://quixdb.github.io/squash-benchmark/?dataset=dickens&machine=s-desktop). - -* [**lzbench**](https://github.com/inikep/lzbench) is an in-memory benchmark of open-source LZ77/LZSS/LZMA compressors. - -* [**fsbench**](https://github.com/gpnuma/fsbench-density) is a command line utility that enables real-time testing of compression algorithms, but also hashes and much more. A fork with density releases is [available here](https://github.com/gpnuma/fsbench-density) for easy access. -The original author's repository [can be found here](https://chiselapp.com/user/Justin_be_my_guide/repository/fsbench/). - -Build ------ -DENSITY can be built on a number of platforms, via the provided makefiles. - -It was developed and optimized against Clang/LLVM which makes it the preferred compiler, but GCC and MSVC are also supported. Please use the latest compiler versions for best performance. - -**MacOS** - -On MacOS, Clang/LLVM is the default compiler, which makes things simpler. - -1) Get the source code : - -``` - git clone https://github.com/k0dai/density.git - cd density -``` - -2) Build and test : - -``` - make - build/benchmark -f -``` - -Alternatively, thanks to the [Homebrew project](https://brew.sh), DENSITY can also be installed with a single command on MacOS: - -``` - brew install density -``` - -**Linux** - -On Linux, Clang/LLVM is not always available by default, but can be easily added thanks to the provided package managers. -The following example assumes a Debian or Ubuntu distribution with *apt-get*. - -1) From the command line, install Clang/LLVM (*optional*, GCC is also supported if Clang/LLVM can't be used) and other prerequisites. - -``` - sudo apt-get install clang git -``` - -2) Get the source code : - -``` - git clone https://github.com/k0dai/density.git - cd density -``` - -3) Build and test : - -``` - make -``` -or -``` - make CC=gcc-... AR=gcc-ar-... -``` -or -``` - make CC=clang-... AR=llvm-ar-... -``` -to choose alternative compilers. For a quick test of resulting binaries, run -``` - build/benchmark -f -``` - -**Windows** - -Please install [git for Windows](https://git-scm.com/download/win) to begin with. - -On Windows, density can be built in different ways. -The **first method** is to use mingw's gcc compiler; for that it is necessary to download and install [mingw-w64](https://sourceforge.net/projects/mingw-w64/). - -1) Once mingw-w64 is installed, get the source : - -``` - git clone https://github.com/k0dai/density.git - cd density -``` - -2) Build and test : - -``` - mingw32-make.exe - build/benchmark.exe -f -``` - -As an alternative, [MSYS2](http://www.msys2.org/) also offers a linux-like environment for Windows. - -The **second method** is to download and install Microsoft's [Visual Studio IDE community edition](https://www.visualstudio.com/thank-you-downloading-visual-studio/?sku=Community). It comes with Microsoft's own compilers and is free. - -1) Once Visual Studio is installed, open a [developer command prompt](https://docs.microsoft.com/en-us/dotnet/framework/tools/developer-command-prompt-for-vs) and type : - -``` - git clone https://github.com/k0dai/density.git - cd density\msvc -``` - -2) Build and test : - -``` - msbuild Density.sln - bin\Release\benchmark.exe -f -``` - -An extra **recommended step** would be to install *Clang/LLVM* for Windows. It is downloadable from [this link](http://releases.llvm.org/5.0.1/LLVM-5.0.1-win64.exe). Once installed, open the Visual Studio IDE by double-clicking on *Density.sln*, then right-click on project names and change the platform toolsets to *LLVM*. Rebuild the solution to generate binaries with Clang/LLVM. - -Output format -------------- -DENSITY outputs compressed data in a simple format, which enables file storage and optional parallelization for both compression and decompression. - -A very short header holding vital informations (like DENSITY version and algorithm used) precedes the binary compressed data. - -APIs ----- -DENSITY features a straightforward *API*, simple yet powerful enough to keep users' creativity unleashed. - -For advanced developers, it allows use of custom dictionaries and exportation of generated dictionaries after a compression session. Although using the default, blank dictionary is perfectly fine in most cases, setting up your own, tailored dictionaries could somewhat improve compression ratio especially for low sized input datum. - -Please see the [*quick start*](#quick-start-a-simple-example-using-the-api) at the bottom of this page. - -About the algorithms --------------------- - -**Chameleon** ( *DENSITY_ALGORITHM_CHAMELEON* ) - -Chameleon is a dictionary lookup based compression algorithm. It is designed for absolute speed and usually reaches a 60% compression ratio on compressible data. -Decompression is just as fast. This algorithm is a great choice when main concern is speed. - -**Cheetah** ( *DENSITY_ALGORITHM_CHEETAH* ) - -Cheetah was developed with inputs from [Piotr Tarsa](https://github.com/tarsa). -It is derived from chameleon and uses swapped double dictionary lookups and predictions. It can be extremely good with highly compressible data (ratio reaching 10% or less). -On typical compressible data compression ratio is about 50% or less. It is still extremely fast for both compression and decompression and is a great, efficient all-rounder algorithm. - -**Lion** ( *DENSITY_ALGORITHM_LION* ) - -Lion is a multiform compression algorithm derived from cheetah. It goes further in the areas of dynamic adaptation and fine-grained analysis. -It uses multiple swapped dictionary lookups and predictions, and forms rank entropy coding. -Lion provides the best compression ratio of all three algorithms under any circumstance, and is still very fast. - -Quick start (a simple example using the API) --------------------------------------------- -Using DENSITY in your application couldn't be any simpler. - -First you need to include this file in your project : - -* density_api.h - -When this is done you can start using the **DENSITY API** : - -```C - #include - #include "density_api.h" - - char* text = "This is a simple example on how to use the simple Density API. This is a simple example on how to use the simple Density API."; - uint64_t text_length = (uint64_t)strlen(text); - - // Determine safe buffer sizes - uint_fast64_t compress_safe_size = density_compress_safe_size(text_length); - uint_fast64_t decompress_safe_size = density_decompress_safe_size(text_length); - - // Allocate required memory - uint8_t *outCompressed = malloc(compress_safe_size * sizeof(char)); - uint8_t *outDecompressed = malloc(decompress_safe_size * sizeof(char)); - density_processing_result result; - - // Compress - result = density_compress(text, text_length, outCompressed, compress_safe_size, DENSITY_COMPRESSION_MODE_CHAMELEON_ALGORITHM); - if(!result.state) - printf("Compressed %llu bytes to %llu bytes\n", result.bytesRead, result.bytesWritten); - - // Decompress - result = density_decompress(outCompressed, result.bytesWritten, outDecompressed, decompress_safe_size); - if(!result.state) - printf("Decompressed %llu bytes to %llu bytes\n", result.bytesRead, result.bytesWritten); - - // Free memory_allocated - free(outCompressed); - free(outDecompressed); -``` - -And that's it ! We've done a compression/decompression round trip with a few lines ! - -Related projects ----------------- - -* **SHARC** (archiver using density algorithms) [https://github.com/gpnuma/sharc](https://github.com/gpnuma/sharc) -* **fsbench-density** (in-memory transformations benchmark) [https://github.com/gpnuma/fsbench-density](https://github.com/gpnuma/fsbench-density) -* **densityxx** (c++ port of density) [https://github.com/charlesw1234/densityxx](https://github.com/charlesw1234/densityxx) diff --git a/contrib/density/appveyor.yml b/contrib/density/appveyor.yml deleted file mode 100644 index 0aabedebc8b..00000000000 --- a/contrib/density/appveyor.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: 0.14.2.{build} -image: Visual Studio 2015 -init: -- cmd: -environment: - matrix: - - EXTRA_PATH: C:\mingw-w64\i686-5.3.0-posix-dwarf-rt_v4-rev0\mingw32\bin\ - LAUNCH_BUILD: mingw32-make CC=gcc AR=gcc-ar - LAUNCH_PATH: build\ - - EXTRA_PATH: C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\ - LAUNCH_BUILD: mingw32-make CC=gcc AR=gcc-ar - LAUNCH_PATH: build\ - - EXTRA_PATH: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\ - LAUNCH_BUILD: mingw32-make CC=gcc AR=gcc-ar - LAUNCH_PATH: build\ - - EXTRA_PATH: C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin\ - LAUNCH_BUILD: mingw32-make CC=gcc AR=gcc-ar - LAUNCH_PATH: build\ - - EXTRA_PATH: '%programfiles(x86)%\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\amd64\' - LAUNCH_BUILD: cd msvc && msbuild.exe Density.sln - EXTRA_GIT: git submodule update --init --recursive - LAUNCH_PATH: bin\Release\ -install: -- cmd: -build_script: -- cmd: >- - %EXTRA_GIT% - - set PATH=%EXTRA_PATH%;%PATH% - - %LAUNCH_BUILD% - - file %LAUNCH_PATH:\=/%* - - %LAUNCH_PATH%benchmark.exe -h -f -test_script: -- cmd: diff --git a/contrib/density/benchmark/src/benchmark.c b/contrib/density/benchmark/src/benchmark.c deleted file mode 100644 index 9781ef361c6..00000000000 --- a/contrib/density/benchmark/src/benchmark.c +++ /dev/null @@ -1,345 +0,0 @@ -/* - * Density benchmark - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * 5/04/15 19:30 - */ - -#include "benchmark.h" - -void density_benchmark_version() { - printf("\nSingle threaded "); - DENSITY_BENCHMARK_BOLD(printf("in-memory benchmark")); - printf(" powered by "); - DENSITY_BENCHMARK_BOLD(printf("Density %i.%i.%i\n", density_version_major(), density_version_minor(), density_version_revision())); - printf("Copyright (C) 2015 Guillaume Voirin\n"); - printf("Built for %s (%s endian system, %u bits) using " DENSITY_BENCHMARK_COMPILER ", %s %s\n", DENSITY_BENCHMARK_PLATFORM_STRING, DENSITY_BENCHMARK_ENDIAN_STRING, (unsigned int) (8 * sizeof(void *)), DENSITY_BENCHMARK_COMPILER_VERSION, __DATE__, __TIME__); -} - -void density_benchmark_client_usage() { - printf("\n"); - DENSITY_BENCHMARK_BOLD(printf("Usage :\n")); - printf(" benchmark [OPTIONS ?]... [FILE ?]\n\n"); - DENSITY_BENCHMARK_BOLD(printf("Available options :\n")); - printf(" -[LEVEL] Test file using only the specified compression LEVEL\n"); - printf(" If unspecified, all algorithms are tested (default).\n"); - printf(" LEVEL can have the following values (as values become higher,\n"); - printf(" compression ratio increases and speed diminishes) :\n"); - printf(" 0 = Copy (no compression)\n"); - printf(" 1 = Chameleon algorithm\n"); - printf(" 2 = Cheetah algorithm\n"); - printf(" 3 = Lion algorithm\n"); - printf(" -c Compress only\n"); - printf(" -f Activate fuzzer mode (pseudorandom generated data)\n"); - printf(" -h Print data hashing informations\n\n"); - exit(EXIT_SUCCESS); -} - -void density_benchmark_format_decimal(uint64_t number) { - if (number < 1000) { - printf("%"PRIu64, number); - return; - } - density_benchmark_format_decimal(number / 1000); - printf(",%03"PRIu64, number % 1000); -} - -const char *density_benchmark_convert_state_to_text(DENSITY_STATE state) { - switch (state) { - case DENSITY_STATE_ERROR_DURING_PROCESSING: - return "Error during processing"; - case DENSITY_STATE_ERROR_INPUT_BUFFER_TOO_SMALL: - return "Input buffer is too small"; - case DENSITY_STATE_ERROR_OUTPUT_BUFFER_TOO_SMALL: - return "Output buffer is too small"; - case DENSITY_STATE_ERROR_INVALID_CONTEXT: - return "Invalid context"; - case DENSITY_STATE_ERROR_INVALID_ALGORITHM: - return "Invalid algorithm"; - default: - return "Unknown error"; - } -} - -int main(int argc, char *argv[]) { - density_benchmark_version(); - DENSITY_ALGORITHM start_mode = DENSITY_ALGORITHM_CHAMELEON; - DENSITY_ALGORITHM end_mode = DENSITY_ALGORITHM_LION; - bool compression_only = false; - bool fuzzer = false; - bool hash_info = false; - char *file_path = NULL; - - if (argc <= 1) - density_benchmark_client_usage(); - for (int count = 1; count < argc; count++) { - if (argv[count][0] == '-') { - switch (argv[count][1]) { - case '1': - start_mode = DENSITY_ALGORITHM_CHAMELEON; - end_mode = DENSITY_ALGORITHM_CHAMELEON; - break; - case '2': - start_mode = DENSITY_ALGORITHM_CHEETAH; - end_mode = DENSITY_ALGORITHM_CHEETAH; - break; - case '3': - start_mode = DENSITY_ALGORITHM_LION; - end_mode = DENSITY_ALGORITHM_LION; - break; - case 'c': - compression_only = true; - break; - case 'f': - fuzzer = true; - break; - case 'h': - hash_info = true; - break; - default: - density_benchmark_client_usage(); - } - } else - file_path = argv[argc - 1]; - } - - uint8_t *in; - uint8_t *out; - uint_fast64_t uncompressed_size; - uint_fast64_t memory_allocated; - if (fuzzer) { - srand((unsigned int) (time(NULL) * 14521937821257379531llu)); - uncompressed_size = (uint_fast64_t) (((uint64_t) (rand() * 100000000llu)) / RAND_MAX); - memory_allocated = density_compress_safe_size(uncompressed_size); - in = malloc(memory_allocated * sizeof(uint8_t)); - uint8_t value = (uint8_t) rand(); - for (unsigned int count = 0; count < uncompressed_size; count++) { - if (!(rand() & 0xf)) - value += (uint8_t)rand(); - in[count] = value; - } - out = malloc(memory_allocated * sizeof(uint8_t)); - } else { - // Open file and get infos - FILE *file = fopen(file_path, "rb"); - if (file == NULL) { - DENSITY_BENCHMARK_ERROR(printf("Error opening file %s.", file_path), false); - } - struct stat file_attributes; - stat(file_path, &file_attributes); - - // Allocate memory and copy file to memory - uncompressed_size = (uint_fast64_t) file_attributes.st_size; - memory_allocated = density_compress_safe_size(uncompressed_size); - in = malloc(memory_allocated * sizeof(uint8_t)); - size_t read = fread(in, sizeof(uint8_t), uncompressed_size, file); - if(uncompressed_size != read) { - DENSITY_BENCHMARK_ERROR(printf("Error reading file %s.", file_path), false); - } - fclose(file); - out = malloc(memory_allocated * sizeof(uint8_t)); - } - - printf("Allocated "); - density_benchmark_format_decimal(2 * memory_allocated); - printf(" bytes of in-memory work space\n"); - - uint64_t original_hash_1 = DENSITY_BENCHMARK_HASH_SEED_1; - uint64_t original_hash_2 = DENSITY_BENCHMARK_HASH_SEED_2; - spookyhash_128(in, uncompressed_size, &original_hash_1, &original_hash_2); - - printf("\n"); - for (DENSITY_ALGORITHM compression_mode = start_mode; compression_mode <= end_mode; compression_mode++) { - // Print algorithm info - switch (compression_mode) { - case DENSITY_ALGORITHM_CHAMELEON: - DENSITY_BENCHMARK_BLUE(DENSITY_BENCHMARK_BOLD(printf("Chameleon algorithm"))); - DENSITY_BENCHMARK_UNDERLINE(19); - break; - case DENSITY_ALGORITHM_CHEETAH: - DENSITY_BENCHMARK_BLUE(DENSITY_BENCHMARK_BOLD(printf("Cheetah algorithm"))); - DENSITY_BENCHMARK_UNDERLINE(17); - break; - case DENSITY_ALGORITHM_LION: - DENSITY_BENCHMARK_BLUE(DENSITY_BENCHMARK_BOLD(printf("Lion algorithm"))); - DENSITY_BENCHMARK_UNDERLINE(14); - break; - } - fflush(stdout); - - // Pre-heat - printf("\nUsing "); - if (fuzzer) { - DENSITY_BENCHMARK_BOLD(printf("generated data")); - } else { - printf("file "); - DENSITY_BENCHMARK_BOLD(printf("%s", file_path)); - } - printf(" copied in memory\n"); - if(hash_info) { - printf("Uncompressed data hash is "); - DENSITY_BENCHMARK_BOLD(printf("0x%" PRIx64 "%" PRIx64, original_hash_1, original_hash_2)); - printf("\n"); - } - - density_processing_result result = density_compress(in, uncompressed_size, out, memory_allocated, compression_mode); - if (result.state) { - DENSITY_BENCHMARK_ERROR(printf("During compress API returned error %i (%s).", result.state, density_benchmark_convert_state_to_text(result.state)), true); - } - const uint_fast64_t compressed_size = result.bytesWritten; - - uint64_t hash_1 = DENSITY_BENCHMARK_HASH_SEED_1; - uint64_t hash_2 = DENSITY_BENCHMARK_HASH_SEED_2; - if(hash_info) { - spookyhash_128(out, compressed_size, &hash_1, &hash_2); - printf("Compressed data hash is "); - DENSITY_BENCHMARK_BOLD(printf("0x%" PRIx64 "%" PRIx64, hash_1, hash_2)); - printf("\n"); - } - - if (!compression_only) { - memset(in, 0, memory_allocated); - result = density_decompress(out, compressed_size, in, memory_allocated); - if (result.state) { - DENSITY_BENCHMARK_ERROR(printf("During decompress API returned error %i (%s).", result.state, density_benchmark_convert_state_to_text(result.state)), true); - } - if (result.bytesWritten != uncompressed_size) { - DENSITY_BENCHMARK_ERROR(printf("Round-trip size differs from original size ("); - density_benchmark_format_decimal(result.bytesWritten); - printf(" bytes against "); - density_benchmark_format_decimal(uncompressed_size); - printf(" bytes).");, true); - } - - hash_1 = DENSITY_BENCHMARK_HASH_SEED_1; - hash_2 = DENSITY_BENCHMARK_HASH_SEED_2; - spookyhash_128(in, uncompressed_size, &hash_1, &hash_2); - - if(hash_info) { - printf("Round-trip data hash is "); - DENSITY_BENCHMARK_BOLD(printf("0x%" PRIx64 "%" PRIx64, hash_1, hash_2)); - printf("\n"); - } - - if(hash_1 != original_hash_1 || hash_2 != original_hash_2) { - DENSITY_BENCHMARK_ERROR(printf("Uncompressed and round-trip data hashes do not match ("); - printf("0x%" PRIx64 "%" PRIx64, hash_1, hash_2); - printf(" vs. "); - printf("0x%" PRIx64 "%" PRIx64, original_hash_1, original_hash_2); - printf(").");, true); - } - - printf("Uncompressed and round-trip data hashes match. "); - } - printf("Starting main bench.\n"); - if (compression_only) - printf("Compressing "); - else - printf("Round-tripping "); - density_benchmark_format_decimal(uncompressed_size); - printf(" bytes to "); - density_benchmark_format_decimal(compressed_size); - printf(" bytes (compression ratio "); - DENSITY_BENCHMARK_BOLD(printf("%.2lf%%", (100.0 * compressed_size) / uncompressed_size)); - printf(" or "); - DENSITY_BENCHMARK_BOLD(printf("%.3fx", (1.0 * uncompressed_size) / compressed_size)); - if (compression_only) - printf(")\n"); - else - printf(") and back\n"); - fflush(stdout); - - // Main benchmark - unsigned int iterations = 0; - double compress_time_high = 0.0; - double compress_time_low = 60.0; - double decompress_time_high = 0.0; - double decompress_time_low = 60.0; - double total_compress_time = 0.0; - double total_decompress_time = 0.0; - double total_time = 0.0; - double decompress_speed = 0.0; - double decompress_speed_low = 0.0; - double decompress_speed_high = 0.0; - double compress_time_elapsed = 0.0; - double decompress_time_elapsed = 0.0; - cputime_chronometer chrono; - - while (total_time <= 10.0) { - ++iterations; - - cputime_chronometer_start(&chrono); - density_compress(in, uncompressed_size, out, memory_allocated, compression_mode); - compress_time_elapsed = cputime_chronometer_stop(&chrono); - - if (!compression_only) { - cputime_chronometer_start(&chrono); - density_decompress(out, compressed_size, in, memory_allocated); - decompress_time_elapsed = cputime_chronometer_stop(&chrono); - } - - total_compress_time += compress_time_elapsed; - - if (compress_time_elapsed < compress_time_low) - compress_time_low = compress_time_elapsed; - if (compress_time_elapsed > compress_time_high) - compress_time_high = compress_time_elapsed; - - double compress_speed = ((1.0 * uncompressed_size * iterations) / (total_compress_time * 1000.0 * 1000.0)); - double compress_speed_low = ((1.0 * uncompressed_size) / (compress_time_high * 1000.0 * 1000.0)); - double compress_speed_high = ((1.0 * uncompressed_size) / (compress_time_low * 1000.0 * 1000.0)); - - total_time += compress_time_elapsed; - - if (!compression_only) { - total_decompress_time += decompress_time_elapsed; - - if (decompress_time_elapsed < decompress_time_low) - decompress_time_low = decompress_time_elapsed; - if (decompress_time_elapsed > decompress_time_high) - decompress_time_high = decompress_time_elapsed; - - decompress_speed = ((1.0 * uncompressed_size * iterations) / (total_decompress_time * 1000.0 * 1000.0)); - decompress_speed_low = ((1.0 * uncompressed_size) / (decompress_time_high * 1000.0 * 1000.0)); - decompress_speed_high = ((1.0 * uncompressed_size) / (decompress_time_low * 1000.0 * 1000.0)); - - total_time += decompress_time_elapsed; - } - - DENSITY_BENCHMARK_BLUE(printf("\rCompress speed "); - DENSITY_BENCHMARK_BOLD(printf("%.0lf MB/s", compress_speed))); - printf(" (min %.0lf MB/s, max %.0lf MB/s, best %.4lfs) ", compress_speed_low, compress_speed_high, compress_time_low); - - if (!compression_only) { - printf("<=> "); - DENSITY_BENCHMARK_BLUE(printf("Decompress speed "); - DENSITY_BENCHMARK_BOLD(printf("%.0lf MB/s", decompress_speed))); - printf(" (min %.0lf MB/s, max %.0lf MB/s, best %.4lfs) ", decompress_speed_low, decompress_speed_high, decompress_time_low); - } - fflush(stdout); - } - printf("\nRun time %.3lfs (%i iterations)\n\n", total_time, iterations); - } - - free(in); - free(out); - - printf("Allocated memory released.\n\n"); - - return EXIT_SUCCESS; -} diff --git a/contrib/density/benchmark/src/benchmark.h b/contrib/density/benchmark/src/benchmark.h deleted file mode 100644 index 749e1ccb1d1..00000000000 --- a/contrib/density/benchmark/src/benchmark.h +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Density benchmark - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * 6/04/15 0:11 - */ - -#ifndef DENSITY_BENCHMARK_H -#define DENSITY_BENCHMARK_H - -#define _CRT_SECURE_NO_DEPRECATE - -#include -#include -#include -#include -#include -#include -#include -#include -#include "../../src/density_api.h" -#include "../libs/cputime/src/cputime_api.h" -#include "../libs/spookyhash/src/spookyhash_api.h" - -#if defined(_WIN64) || defined(_WIN32) -#else -#define DENSITY_BENCHMARK_ALLOW_ANSI_ESCAPE_SEQUENCES -#endif - -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define DENSITY_BENCHMARK_ENDIAN_STRING "Little" -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -#define DENSITY_BENCHMARK_ENDIAN_STRING "Big" -#endif - -#if defined(__clang__) -#define DENSITY_BENCHMARK_COMPILER "Clang %d.%d.%d" -#define DENSITY_BENCHMARK_COMPILER_VERSION __clang_major__, __clang_minor__, __clang_patchlevel__ -#elif defined(__GNUC__) -#define DENSITY_BENCHMARK_COMPILER "GCC %d.%d.%d" -#define DENSITY_BENCHMARK_COMPILER_VERSION __GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__ -#elif defined(_MSC_VER) -#define DENSITY_BENCHMARK_COMPILER "MSVC" -#define DENSITY_BENCHMARK_COMPILER_VERSION "" -#elif defined(__INTEL_COMPILER) -#define DENSITY_BENCHMARK_COMPILER "ICC" -#define DENSITY_BENCHMARK_COMPILER_VERSION "" -#else -#define DENSITY_BENCHMARK_COMPILER "an unknown compiler" -#define DENSITY_BENCHMARK_COMPILER_VERSION "" -#endif - -#if defined(_WIN64) || defined(_WIN32) -#define DENSITY_BENCHMARK_PLATFORM_STRING "Microsoft Windows" -#elif defined(__APPLE__) -#include "TargetConditionals.h" -#if TARGET_IPHONE_SIMULATOR -#define DENSITY_BENCHMARK_PLATFORM_STRING "iOS Simulator" -#elif TARGET_OS_IPHONE -#define DENSITY_BENCHMARK_PLATFORM_STRING "iOS" -#elif TARGET_OS_MAC -#define DENSITY_BENCHMARK_PLATFORM_STRING "MacOS" -#else -#define DENSITY_BENCHMARK_PLATFORM_STRING "an unknown Apple platform" -#endif -#elif defined(__FreeBSD__) -#define DENSITY_BENCHMARK_PLATFORM_STRING "FreeBSD" -#elif defined(__linux__) -#define DENSITY_BENCHMARK_PLATFORM_STRING "GNU/Linux" -#elif defined(__unix__) -#define DENSITY_BENCHMARK_PLATFORM_STRING "Unix" -#elif defined(__posix__) -#define DENSITY_BENCHMARK_PLATFORM_STRING "Posix" -#else -#define DENSITY_BENCHMARK_PLATFORM_STRING "an unknown platform" -#endif - -#define DENSITY_ESCAPE_CHARACTER ((char)27) - -#ifdef DENSITY_BENCHMARK_ALLOW_ANSI_ESCAPE_SEQUENCES -#define DENSITY_BENCHMARK_BOLD(op) printf("%c[1m", DENSITY_ESCAPE_CHARACTER);\ - op;\ - printf("%c[0m", DENSITY_ESCAPE_CHARACTER); - -#define DENSITY_BENCHMARK_BLUE(op) printf("%c[0;34m", DENSITY_ESCAPE_CHARACTER);\ - op;\ - printf("%c[0m", DENSITY_ESCAPE_CHARACTER); - -#define DENSITY_BENCHMARK_RED(op) printf("%c[0;31m", DENSITY_ESCAPE_CHARACTER);\ - op;\ - printf("%c[0m", DENSITY_ESCAPE_CHARACTER); -#else -#define DENSITY_BENCHMARK_BOLD(op) op; -#define DENSITY_BENCHMARK_BLUE(op) op; -#define DENSITY_BENCHMARK_RED(op) op; -#endif - -#define DENSITY_BENCHMARK_UNDERLINE(n) printf("\n");\ - for(int i = 0; i < n; i++) printf("="); - -#define DENSITY_BENCHMARK_ERROR(op, issue) DENSITY_BENCHMARK_RED(DENSITY_BENCHMARK_BOLD(printf("\nAn error has occured !\n")));\ - op;\ - printf("\n");\ - if(issue) {\ - printf("Please open an issue at , with your platform information and any relevant file.\n");\ - DENSITY_BENCHMARK_BOLD(printf("Thank you !\n"));\ - }\ - fflush(stdout);\ - exit(EXIT_FAILURE); - -#endif - -#define DENSITY_BENCHMARK_HASH_SEED_1 0x0123456789abcdefllu -#define DENSITY_BENCHMARK_HASH_SEED_2 0xfedcba9876543210llu diff --git a/contrib/density/msvc/Density.sln b/contrib/density/msvc/Density.sln deleted file mode 100644 index 70c25ab3413..00000000000 --- a/contrib/density/msvc/Density.sln +++ /dev/null @@ -1,21 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "density", "density.vcxproj", "{65C51F09-D1A4-9EA4-DABC-297B461B0506}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "benchmark", "benchmark.vcxproj", "{7000C5C1-DC6A-7938-25A9-2ADE9152578D}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Release|x64 = Release|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {65C51F09-D1A4-9EA4-DABC-297B461B0506}.Release|x64.ActiveCfg = Release|x64 - {65C51F09-D1A4-9EA4-DABC-297B461B0506}.Release|x64.Build.0 = Release|x64 - {7000C5C1-DC6A-7938-25A9-2ADE9152578D}.Release|x64.ActiveCfg = Release|x64 - {7000C5C1-DC6A-7938-25A9-2ADE9152578D}.Release|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/contrib/density/msvc/benchmark.vcxproj b/contrib/density/msvc/benchmark.vcxproj deleted file mode 100644 index ef32bbb5f3c..00000000000 --- a/contrib/density/msvc/benchmark.vcxproj +++ /dev/null @@ -1,78 +0,0 @@ - - - - - Release - x64 - - - - {7000C5C1-DC6A-7938-25A9-2ADE9152578D} - true - Win32Proj - benchmark - - - - Application - false - Unicode - v140 - true - - - - - - - - - - false - bin\Release\ - obj\Release\benchmark\ - benchmark - .exe - - - - NotUsing - Level4 - MaxSpeed - true - true - false - true - true - - - Console - true - true - - - - - - - - - - - - - - - - - - - - - {65C51F09-D1A4-9EA4-DABC-297B461B0506} - - - - - - \ No newline at end of file diff --git a/contrib/density/msvc/benchmark.vcxproj.filters b/contrib/density/msvc/benchmark.vcxproj.filters deleted file mode 100644 index 26379e28724..00000000000 --- a/contrib/density/msvc/benchmark.vcxproj.filters +++ /dev/null @@ -1,63 +0,0 @@ - - - - - {2F149A7C-1B4B-9B0D-C437-8110B04D170F} - - - {95C9EAC4-812C-7A69-2AB2-B21F16F445EC} - - - {2C4ACB69-1843-EABB-4175-CF402DCDC9C7} - - - {C78AF53E-3316-6303-3C27-E6F7A831BF03} - - - {5E54D780-CAB5-B48E-5323-FB40BF83EB4D} - - - {2DAB880B-99B4-887C-2230-9F7C8E38947C} - - - - - libs\cputime\src - - - libs\cputime\src - - - libs\spookyhash\src - - - libs\spookyhash\src - - - libs\spookyhash\src - - - libs\spookyhash\src - - - src - - - - - libs\cputime\src - - - libs\spookyhash\src - - - libs\spookyhash\src - - - libs\spookyhash\src - - - src - - - \ No newline at end of file diff --git a/contrib/density/msvc/density.vcxproj b/contrib/density/msvc/density.vcxproj deleted file mode 100644 index f1ad77fbc1b..00000000000 --- a/contrib/density/msvc/density.vcxproj +++ /dev/null @@ -1,93 +0,0 @@ - - - - - Release - x64 - - - - {65C51F09-D1A4-9EA4-DABC-297B461B0506} - true - Win32Proj - density - - - - DynamicLibrary - false - Unicode - v140 - true - - - - - - - - - - false - bin\Release\ - obj\Release\density\ - density - .dll - - - - NotUsing - Level4 - MaxSpeed - true - true - false - true - true - - - Windows - true - true - bin\Release\density.lib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/contrib/density/msvc/density.vcxproj.filters b/contrib/density/msvc/density.vcxproj.filters deleted file mode 100644 index 202f2522bbe..00000000000 --- a/contrib/density/msvc/density.vcxproj.filters +++ /dev/null @@ -1,135 +0,0 @@ - - - - - {DF10C562-CBC8-06B0-34D0-DF0B20A7A1A0} - - - {5AB95D49-4648-E712-EF66-FB0DDBD4F7B8} - - - {1252540B-7E14-8895-C7C4-162233C66073} - - - {3F6054BC-AB43-63FC-B446-913820A9294D} - - - {C09C0CB6-AC80-CD0B-15E1-C75E01E4B78D} - - - {3845B705-A47C-FF9C-AD86-63C0193D2696} - - - {E597066D-51B0-ED96-1A5D-7D3086348230} - - - {40FA2C44-AC85-9A08-B596-1DFD21A1F608} - - - {B8122E82-A4A1-B74B-4DC0-CB46392EC8F1} - - - {6585C1C9-5195-6D9A-BA4E-D2B8A67D5C33} - - - {B6B426C8-2221-E2C2-EB14-7A205740042B} - - - {B2DFE593-1EBF-642F-27D7-EF059335CB90} - - - {367A73B3-A2E4-272A-EB22-D9CF57CC057F} - - - - - algorithms - - - algorithms\chameleon - - - algorithms\chameleon\core - - - algorithms\chameleon\core - - - algorithms\chameleon\dictionary - - - algorithms\cheetah - - - algorithms\cheetah\core - - - algorithms\cheetah\core - - - algorithms\cheetah\dictionary - - - algorithms - - - algorithms\lion\core - - - algorithms\lion\core - - - algorithms\lion\dictionary - - - algorithms\lion\forms - - - algorithms\lion - - - buffers - - - - - structure - - - - - algorithms - - - algorithms\chameleon\core - - - algorithms\chameleon\core - - - algorithms\cheetah\core - - - algorithms\cheetah\core - - - algorithms - - - algorithms\lion\core - - - algorithms\lion\core - - - algorithms\lion\forms - - - buffers - - - - structure - - - \ No newline at end of file diff --git a/contrib/density/src/algorithms/algorithms.c b/contrib/density/src/algorithms/algorithms.c deleted file mode 100644 index f36e94cd08b..00000000000 --- a/contrib/density/src/algorithms/algorithms.c +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Density - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 14/10/15 02:06 - */ - -#include "algorithms.h" - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_algorithms_prepare_state(density_algorithm_state *const DENSITY_RESTRICT state, void *const DENSITY_RESTRICT dictionary) { - state->dictionary = dictionary; - state->copy_penalty = 0; - state->copy_penalty_start = 1; - state->previous_incompressible = false; - state->counter = 0; -} diff --git a/contrib/density/src/algorithms/algorithms.h b/contrib/density/src/algorithms/algorithms.h deleted file mode 100644 index b5e31377004..00000000000 --- a/contrib/density/src/algorithms/algorithms.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Density - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 3/02/15 19:51 - */ - -#ifndef DENSITY_ALGORITHMS_H -#define DENSITY_ALGORITHMS_H - -#include "../globals.h" - -typedef enum { - DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED = 0, - DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING, - DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL, - DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL -} density_algorithm_exit_status; - -typedef struct { - void *dictionary; - uint_fast8_t copy_penalty; - uint_fast8_t copy_penalty_start; - bool previous_incompressible; - uint_fast64_t counter; -} density_algorithm_state; - -#define DENSITY_ALGORITHM_COPY(work_block_size)\ - DENSITY_MEMCPY(*out, *in, work_block_size);\ - *in += work_block_size;\ - *out += work_block_size; - -#define DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START\ - if(!(--state->copy_penalty))\ - state->copy_penalty_start++; - -#define DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START\ - if (state->copy_penalty_start & ~0x1)\ - state->copy_penalty_start >>= 1; - -#define DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY(span, work_block_size)\ - if (DENSITY_UNLIKELY(span & ~(work_block_size - 1))) {\ - if (state->previous_incompressible)\ - state->copy_penalty = state->copy_penalty_start;\ - state->previous_incompressible = true;\ - } else\ - state->previous_incompressible = false; - -DENSITY_WINDOWS_EXPORT void density_algorithms_prepare_state(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, void *const DENSITY_RESTRICT_DECLARE); - -#endif diff --git a/contrib/density/src/algorithms/chameleon/chameleon.h b/contrib/density/src/algorithms/chameleon/chameleon.h deleted file mode 100644 index 2054f34cfd8..00000000000 --- a/contrib/density/src/algorithms/chameleon/chameleon.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 24/10/13 11:57 - * - * ------------------- - * Chameleon algorithm - * ------------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Hash based superfast kernel - */ - -#ifndef DENSITY_CHAMELEON_H -#define DENSITY_CHAMELEON_H - -#include "../../globals.h" - -#define DENSITY_CHAMELEON_HASH_BITS 16 -#define DENSITY_CHAMELEON_HASH_MULTIPLIER (uint32_t)0x9D6EF916lu - -#define DENSITY_CHAMELEON_HASH_ALGORITHM(value32) (uint16_t)((value32 * DENSITY_CHAMELEON_HASH_MULTIPLIER) >> (32 - DENSITY_CHAMELEON_HASH_BITS)) - -typedef enum { - DENSITY_CHAMELEON_SIGNATURE_FLAG_CHUNK = 0x0, - DENSITY_CHAMELEON_SIGNATURE_FLAG_MAP = 0x1, -} DENSITY_CHAMELEON_SIGNATURE_FLAG; - -typedef uint64_t density_chameleon_signature; - -#define DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE (density_bitsizeof(density_chameleon_signature) * sizeof(uint32_t)) // Uncompressed chunks -#define DENSITY_CHAMELEON_DECOMPRESSED_BODY_SIZE_PER_SIGNATURE (density_bitsizeof(density_chameleon_signature) * sizeof(uint32_t)) - -#define DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE (sizeof(density_chameleon_signature) + DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE) -#define DENSITY_CHAMELEON_DECOMPRESSED_UNIT_SIZE (DENSITY_CHAMELEON_DECOMPRESSED_BODY_SIZE_PER_SIGNATURE) - -#define DENSITY_CHAMELEON_WORK_BLOCK_SIZE 256 - -#endif diff --git a/contrib/density/src/algorithms/chameleon/core/chameleon_decode.c b/contrib/density/src/algorithms/chameleon/core/chameleon_decode.c deleted file mode 100644 index 240aaba8598..00000000000 --- a/contrib/density/src/algorithms/chameleon/core/chameleon_decode.c +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 23/06/15 22:11 - * - * ------------------- - * Chameleon algorithm - * ------------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Hash based superfast kernel - */ - -#include "chameleon_decode.h" - -DENSITY_FORCE_INLINE void density_chameleon_decode_process_compressed(const uint16_t hash, uint8_t **DENSITY_RESTRICT out, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) { - DENSITY_MEMCPY(*out, &dictionary->entries[hash].as_uint32_t, sizeof(uint32_t)); -} - -DENSITY_FORCE_INLINE void density_chameleon_decode_process_uncompressed(const uint32_t chunk, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) { - const uint16_t hash = DENSITY_CHAMELEON_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(chunk)); - (&dictionary->entries[hash])->as_uint32_t = chunk; // Does not ensure dictionary content consistency between endiannesses -} - -DENSITY_FORCE_INLINE void density_chameleon_decode_kernel(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const density_bool compressed, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) { - if (compressed) { - uint16_t hash; - DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t)); - density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16(hash), out, dictionary); - *in += sizeof(uint16_t); - } else { - uint32_t unit; - DENSITY_MEMCPY(&unit, *in, sizeof(uint32_t)); - density_chameleon_decode_process_uncompressed(unit, dictionary); - DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t)); - *in += sizeof(uint32_t); - } - *out += sizeof(uint32_t); -} - -DENSITY_FORCE_INLINE void density_chameleon_decode_kernel_dual(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const density_chameleon_signature signature, const uint_fast8_t shift, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) { - uint32_t var_32; - uint64_t var_64; - - switch((signature >> shift) & 0x3) { - case 0x0: - DENSITY_MEMCPY(&var_64, *in, sizeof(uint32_t) + sizeof(uint32_t)); -#ifdef DENSITY_LITTLE_ENDIAN - density_chameleon_decode_process_uncompressed((uint32_t)(var_64 & 0xffffffff), dictionary); -#endif - density_chameleon_decode_process_uncompressed((uint32_t)(var_64 >> density_bitsizeof(uint32_t)), dictionary); -#ifdef DENSITY_BIG_ENDIAN - density_chameleon_decode_process_uncompressed((uint32_t)(var_64 & 0xffffffff), dictionary); -#endif - DENSITY_MEMCPY(*out, &var_64, sizeof(uint32_t) + sizeof(uint32_t)); - *in += (sizeof(uint32_t) + sizeof(uint32_t)); - *out += sizeof(uint64_t); - break; - case 0x1: - DENSITY_MEMCPY(&var_64, *in, sizeof(uint16_t) + sizeof(uint32_t)); -#ifdef DENSITY_LITTLE_ENDIAN - density_chameleon_decode_process_compressed((uint16_t)(var_64 & 0xffff), out, dictionary); - var_32 = (uint32_t)((var_64 >> density_bitsizeof(uint16_t)) & 0xffffffff); - density_chameleon_decode_process_uncompressed(var_32, dictionary); - DENSITY_MEMCPY(*out + sizeof(uint32_t), &var_32, sizeof(uint32_t)); - *out += sizeof(uint64_t); -#elif defined(DENSITY_BIG_ENDIAN) - density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16((uint16_t)((var_64 >> (density_bitsizeof(uint16_t) + density_bitsizeof(uint32_t))) & 0xffff)), out, dictionary); - var_32 = (uint32_t)((var_64 >> density_bitsizeof(uint16_t)) & 0xffffffff); - density_chameleon_decode_process_uncompressed(var_32, dictionary); - DENSITY_MEMCPY(*out + sizeof(uint32_t), &var_32, sizeof(uint32_t)); - *out += sizeof(uint64_t); -#else -#error -#endif - *in += (sizeof(uint16_t) + sizeof(uint32_t)); - break; - case 0x2: - DENSITY_MEMCPY(&var_64, *in, sizeof(uint32_t) + sizeof(uint16_t)); -#ifdef DENSITY_LITTLE_ENDIAN - var_32 = (uint32_t)(var_64 & 0xffffffff); - density_chameleon_decode_process_uncompressed(var_32, dictionary); - DENSITY_MEMCPY(*out, &var_32, sizeof(uint32_t)); - *out += sizeof(uint32_t); - density_chameleon_decode_process_compressed((uint16_t)((var_64 >> density_bitsizeof(uint32_t)) & 0xffff), out, dictionary); - *out += sizeof(uint32_t); -#elif defined(DENSITY_BIG_ENDIAN) - var_32 = (uint32_t)((var_64 >> density_bitsizeof(uint32_t)) & 0xffffffff); - density_chameleon_decode_process_uncompressed(var_32, dictionary); - DENSITY_MEMCPY(*out, &var_32, sizeof(uint32_t)); - *out += sizeof(uint32_t); - density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16((uint16_t)((var_64 >> density_bitsizeof(uint16_t)) & 0xffff)), out, dictionary); - *out += sizeof(uint32_t); -#else -#error -#endif - *in += (sizeof(uint32_t) + sizeof(uint16_t)); - break; - case 0x3: - DENSITY_MEMCPY(&var_32, *in, sizeof(uint16_t) + sizeof(uint16_t)); -#ifdef DENSITY_LITTLE_ENDIAN - density_chameleon_decode_process_compressed((uint16_t)(var_32 & 0xffff), out, dictionary); - *out += sizeof(uint32_t); -#endif - density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16((uint16_t)(var_32 >> density_bitsizeof(uint16_t))), out, dictionary); - *out += sizeof(uint32_t); -#ifdef DENSITY_BIG_ENDIAN - density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16((uint16_t)(var_32 & 0xffff)), out, dictionary); - *out += sizeof(uint32_t); -#endif - *in += (sizeof(uint16_t) + sizeof(uint16_t)); - break; - } -} - -DENSITY_FORCE_INLINE bool density_chameleon_decode_test_compressed(const density_chameleon_signature signature, const uint_fast8_t shift) { - return (density_bool const) ((signature >> shift) & DENSITY_CHAMELEON_SIGNATURE_FLAG_MAP); -} - -DENSITY_FORCE_INLINE void density_chameleon_decode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const density_chameleon_signature signature, const uint_fast8_t shift, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) { - density_chameleon_decode_kernel(in, out, density_chameleon_decode_test_compressed(signature, shift), dictionary); -} - -DENSITY_FORCE_INLINE void density_chameleon_decode_256(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const density_chameleon_signature signature, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) { - uint_fast8_t count_a = 0; - uint_fast8_t count_b = 0; - -#if defined(__clang__) || defined(_MSC_VER) - do { - DENSITY_UNROLL_2(density_chameleon_decode_kernel_dual(in, out, signature, count_a, dictionary); count_a+= 2); - } while (++count_b & 0xf); -#else - do { - DENSITY_UNROLL_2(density_chameleon_decode_4(in, out, signature, count_a ++, dictionary)); - } while (++count_b & 0x1f); -#endif -} - -DENSITY_FORCE_INLINE void density_chameleon_decode_read_signature(const uint8_t **DENSITY_RESTRICT in, density_chameleon_signature *DENSITY_RESTRICT signature) { -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(signature, *in, sizeof(density_chameleon_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - density_chameleon_signature endian_signature; - DENSITY_MEMCPY(&endian_signature, *in, sizeof(density_chameleon_signature)); - *signature = DENSITY_LITTLE_ENDIAN_64(endian_signature); -#else -#error -#endif - *in += sizeof(density_chameleon_signature); -} - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_chameleon_decode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) { - if (out_size < DENSITY_CHAMELEON_DECOMPRESSED_UNIT_SIZE) - return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; - - density_chameleon_signature signature; - uint_fast8_t shift; - uint_fast64_t remaining; - - const uint8_t *start = *in; - - if (in_size < DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE) { - goto read_signature; - } - - const uint8_t *in_limit = *in + in_size - DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE; - uint8_t *out_limit = *out + out_size - DENSITY_CHAMELEON_DECOMPRESSED_UNIT_SIZE; - - while (DENSITY_LIKELY(*in <= in_limit && *out <= out_limit)) { - if (DENSITY_UNLIKELY(!(state->counter & 0xf))) { - DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START; - } - state->counter++; - if (DENSITY_UNLIKELY(state->copy_penalty)) { - DENSITY_ALGORITHM_COPY(DENSITY_CHAMELEON_WORK_BLOCK_SIZE); - DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START; - } else { - const uint8_t *in_start = *in; - density_chameleon_decode_read_signature(in, &signature); - density_chameleon_decode_256(in, out, signature, (density_chameleon_dictionary *const) state->dictionary); - DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*in - in_start), DENSITY_CHAMELEON_WORK_BLOCK_SIZE); - } - } - - if (*out > out_limit) - return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; - - read_signature: - if (in_size - (*in - start) < sizeof(density_chameleon_signature)) - return DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL; - shift = 0; - density_chameleon_decode_read_signature(in, &signature); - read_and_decode_4: - switch (in_size - (*in - start)) { - case 0: - case 1: - if (density_chameleon_decode_test_compressed(signature, shift)) - return DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING; - else // End marker - goto process_remaining_bytes; - case 2: - case 3: - if (density_chameleon_decode_test_compressed(signature, shift++)) - density_chameleon_decode_kernel(in, out, true, (density_chameleon_dictionary *const) state->dictionary); - else // End marker - goto process_remaining_bytes; - break; - default: - density_chameleon_decode_4(in, out, signature, shift++, (density_chameleon_dictionary *const) state->dictionary); - break; - } - - if (DENSITY_UNLIKELY(shift == density_bitsizeof(density_chameleon_signature))) - goto read_signature; - else - goto read_and_decode_4; - - process_remaining_bytes: - remaining = in_size - (*in - start); - DENSITY_ALGORITHM_COPY(remaining); - - return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED; -} diff --git a/contrib/density/src/algorithms/chameleon/core/chameleon_decode.h b/contrib/density/src/algorithms/chameleon/core/chameleon_decode.h deleted file mode 100644 index cbf06e26a44..00000000000 --- a/contrib/density/src/algorithms/chameleon/core/chameleon_decode.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 23/06/15 21:49 - * - * ------------------- - * Chameleon algorithm - * ------------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Hash based superfast kernel - */ - -#ifndef DENSITY_CHAMELEON_DECODE_H -#define DENSITY_CHAMELEON_DECODE_H - -#include "../dictionary/chameleon_dictionary.h" -#include "../../algorithms.h" - -DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_chameleon_decode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t); - -#endif diff --git a/contrib/density/src/algorithms/chameleon/core/chameleon_encode.c b/contrib/density/src/algorithms/chameleon/core/chameleon_encode.c deleted file mode 100644 index 2090cb99173..00000000000 --- a/contrib/density/src/algorithms/chameleon/core/chameleon_encode.c +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 23/06/15 22:02 - * - * ------------------- - * Chameleon algorithm - * ------------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Hash based superfast kernel - */ - -#include "chameleon_encode.h" - -DENSITY_FORCE_INLINE void density_chameleon_encode_prepare_signature(uint8_t **DENSITY_RESTRICT out, density_chameleon_signature **DENSITY_RESTRICT signature_pointer, density_chameleon_signature *const DENSITY_RESTRICT signature) { - *signature = 0; - *signature_pointer = (density_chameleon_signature *) *out; - *out += sizeof(density_chameleon_signature); -} - -DENSITY_FORCE_INLINE void density_chameleon_encode_kernel(uint8_t **DENSITY_RESTRICT out, const uint16_t hash, const uint_fast8_t shift, density_chameleon_signature *const DENSITY_RESTRICT signature, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) { - density_chameleon_dictionary_entry *const found = &dictionary->entries[hash]; - - switch (*unit ^ found->as_uint32_t) { - case 0: - *signature |= ((uint64_t) DENSITY_CHAMELEON_SIGNATURE_FLAG_MAP << shift); -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); -#elif defined(DENSITY_BIG_ENDIAN) - const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); - DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); -#else -#error -#endif - *out += sizeof(uint16_t); - break; - default: - found->as_uint32_t = *unit; // Does not ensure dictionary content consistency between endiannesses - DENSITY_MEMCPY(*out, unit, sizeof(uint32_t)); - *out += sizeof(uint32_t); - break; - } -} - -DENSITY_FORCE_INLINE void density_chameleon_encode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const uint_fast8_t shift, density_chameleon_signature *const DENSITY_RESTRICT signature, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) { - DENSITY_MEMCPY(unit, *in, sizeof(uint32_t)); - density_chameleon_encode_kernel(out, DENSITY_CHAMELEON_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)), shift, signature, dictionary, unit); - *in += sizeof(uint32_t); -} - -DENSITY_FORCE_INLINE void density_chameleon_encode_256(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, density_chameleon_signature *const DENSITY_RESTRICT signature, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) { - uint_fast8_t count = 0; - -#ifdef __clang__ - for (uint_fast8_t count_b = 0; count_b < 32; count_b++) { - DENSITY_UNROLL_2(density_chameleon_encode_4(in, out, count++, signature, dictionary, unit)); - } -#else - for (uint_fast8_t count_b = 0; count_b < 16; count_b++) { - DENSITY_UNROLL_4(density_chameleon_encode_4(in, out, count++, signature, dictionary, unit)); - } -#endif -} - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_chameleon_encode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) { - if (out_size < DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE) - return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; - - density_chameleon_signature signature; - density_chameleon_signature *signature_pointer; - uint32_t unit; - - uint8_t *out_limit = *out + out_size - DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE; - uint_fast64_t limit_256 = (in_size >> 8); - - while (DENSITY_LIKELY(limit_256-- && *out <= out_limit)) { - if (DENSITY_UNLIKELY(!(state->counter & 0xf))) { - DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START; - } - state->counter++; - if (DENSITY_UNLIKELY(state->copy_penalty)) { - DENSITY_ALGORITHM_COPY(DENSITY_CHAMELEON_WORK_BLOCK_SIZE); - DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START; - } else { - const uint8_t *out_start = *out; - density_chameleon_encode_prepare_signature(out, &signature_pointer, &signature); - DENSITY_PREFETCH(*in + DENSITY_CHAMELEON_WORK_BLOCK_SIZE); - density_chameleon_encode_256(in, out, &signature, (density_chameleon_dictionary *const) state->dictionary, &unit); -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_chameleon_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - const density_chameleon_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); - DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_chameleon_signature)); -#else -#error -#endif - DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*out - out_start), DENSITY_CHAMELEON_WORK_BLOCK_SIZE); - } - } - - if (*out > out_limit) - return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; - - uint_fast64_t remaining; - - switch (in_size & 0xff) { - case 0: - case 1: - case 2: - case 3: - density_chameleon_encode_prepare_signature(out, &signature_pointer, &signature); - signature = ((uint64_t) DENSITY_CHAMELEON_SIGNATURE_FLAG_CHUNK); // End marker -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_chameleon_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - const density_chameleon_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); - DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_chameleon_signature)); -#else -#error -#endif - goto process_remaining_bytes; - default: - break; - } - - const uint_fast64_t limit_4 = (in_size & 0xff) >> 2; - density_chameleon_encode_prepare_signature(out, &signature_pointer, &signature); - for (uint_fast8_t shift = 0; shift != limit_4; shift++) - density_chameleon_encode_4(in, out, shift, &signature, (density_chameleon_dictionary *const) state->dictionary, &unit); - - signature |= ((uint64_t) DENSITY_CHAMELEON_SIGNATURE_FLAG_CHUNK << limit_4); // End marker -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_chameleon_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - const density_chameleon_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); - DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_chameleon_signature)); -#else -#error -#endif - - process_remaining_bytes: - remaining = in_size & 0x3; - if (remaining) - DENSITY_ALGORITHM_COPY(remaining); - - return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED; -} diff --git a/contrib/density/src/algorithms/chameleon/core/chameleon_encode.h b/contrib/density/src/algorithms/chameleon/core/chameleon_encode.h deleted file mode 100644 index f71fc23fbc0..00000000000 --- a/contrib/density/src/algorithms/chameleon/core/chameleon_encode.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 23/06/15 21:51 - * - * ------------------- - * Chameleon algorithm - * ------------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Hash based superfast kernel - */ - -#ifndef DENSITY_CHAMELEON_ENCODE_H -#define DENSITY_CHAMELEON_ENCODE_H - -#include "../dictionary/chameleon_dictionary.h" -#include "../../algorithms.h" - -DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_chameleon_encode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t); - -#endif diff --git a/contrib/density/src/algorithms/chameleon/dictionary/chameleon_dictionary.h b/contrib/density/src/algorithms/chameleon/dictionary/chameleon_dictionary.h deleted file mode 100644 index 1f584a5936e..00000000000 --- a/contrib/density/src/algorithms/chameleon/dictionary/chameleon_dictionary.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 24/10/13 12:05 - * - * ------------------- - * Chameleon algorithm - * ------------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Hash based superfast kernel - */ - -#ifndef DENSITY_CHAMELEON_DICTIONARY_H -#define DENSITY_CHAMELEON_DICTIONARY_H - -#include "../chameleon.h" - -#include - -#pragma pack(push) -#pragma pack(4) -typedef struct { - uint32_t as_uint32_t; -} density_chameleon_dictionary_entry; - -typedef struct { - density_chameleon_dictionary_entry entries[1 << DENSITY_CHAMELEON_HASH_BITS]; -} density_chameleon_dictionary; -#pragma pack(pop) - -#endif diff --git a/contrib/density/src/algorithms/cheetah/cheetah.h b/contrib/density/src/algorithms/cheetah/cheetah.h deleted file mode 100644 index 0d47d0ab502..00000000000 --- a/contrib/density/src/algorithms/cheetah/cheetah.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 06/12/13 20:10 - * - * ----------------- - * Cheetah algorithm - * ----------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * Piotr Tarsa (https://github.com/tarsa) - * - * Description - * Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup - */ - -#ifndef DENSITY_CHEETAH_H -#define DENSITY_CHEETAH_H - -#include "../../globals.h" - -#define DENSITY_CHEETAH_HASH_BITS 16 -#define DENSITY_CHEETAH_HASH_MULTIPLIER (uint32_t)0x9D6EF916lu - -#define DENSITY_CHEETAH_HASH_ALGORITHM(value32) (uint16_t)(value32 * DENSITY_CHEETAH_HASH_MULTIPLIER >> (32 - DENSITY_CHEETAH_HASH_BITS)) - -typedef enum { - DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED = 0x0, - DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_A = 0x1, - DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_B = 0x2, - DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK = 0x3, -} DENSITY_CHEETAH_SIGNATURE_FLAG; - -typedef uint64_t density_cheetah_signature; - -#define DENSITY_CHEETAH_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE ((density_bitsizeof(density_cheetah_signature) >> 1) * sizeof(uint32_t)) // Uncompressed chunks -#define DENSITY_CHEETAH_DECOMPRESSED_BODY_SIZE_PER_SIGNATURE ((density_bitsizeof(density_cheetah_signature) >> 1) * sizeof(uint32_t)) - -#define DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE (sizeof(density_cheetah_signature) + DENSITY_CHEETAH_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE) -#define DENSITY_CHEETAH_DECOMPRESSED_UNIT_SIZE (DENSITY_CHEETAH_DECOMPRESSED_BODY_SIZE_PER_SIGNATURE) - -#define DENSITY_CHEETAH_WORK_BLOCK_SIZE 128 - -#endif \ No newline at end of file diff --git a/contrib/density/src/algorithms/cheetah/core/cheetah_decode.c b/contrib/density/src/algorithms/cheetah/core/cheetah_decode.c deleted file mode 100644 index 4928938b41e..00000000000 --- a/contrib/density/src/algorithms/cheetah/core/cheetah_decode.c +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 24/06/15 0:32 - * - * ----------------- - * Cheetah algorithm - * ----------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * Piotr Tarsa (https://github.com/tarsa) - * - * Description - * Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup - */ - -#include "cheetah_decode.h" - -DENSITY_FORCE_INLINE void density_cheetah_decode_process_predicted(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) { - const uint32_t unit = dictionary->prediction_entries[*last_hash].next_chunk_prediction; - DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t)); - *last_hash = DENSITY_CHEETAH_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(unit)); -} - -DENSITY_FORCE_INLINE void density_cheetah_decode_process_compressed_a(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, const uint16_t hash) { - DENSITY_PREFETCH(&dictionary->prediction_entries[hash]); - const uint32_t unit = dictionary->entries[hash].chunk_a; - DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t)); - dictionary->prediction_entries[*last_hash].next_chunk_prediction = unit; - *last_hash = hash; -} - -DENSITY_FORCE_INLINE void density_cheetah_decode_process_compressed_b(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, const uint16_t hash) { - DENSITY_PREFETCH(&dictionary->prediction_entries[hash]); - density_cheetah_dictionary_entry *const entry = &dictionary->entries[hash]; - const uint32_t unit = entry->chunk_b; - entry->chunk_b = entry->chunk_a; - entry->chunk_a = unit; // Does not ensure dictionary content consistency between endiannesses - DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t)); - dictionary->prediction_entries[*last_hash].next_chunk_prediction = unit; - *last_hash = hash; -} - -DENSITY_FORCE_INLINE void density_cheetah_decode_process_uncompressed(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, const uint32_t unit) { - const uint16_t hash = DENSITY_CHEETAH_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(unit)); - DENSITY_PREFETCH(&dictionary->prediction_entries[hash]); - density_cheetah_dictionary_entry *const entry = &dictionary->entries[hash]; - entry->chunk_b = entry->chunk_a; - entry->chunk_a = unit; // Does not ensure dictionary content consistency between endiannesses - DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t)); - dictionary->prediction_entries[*last_hash].next_chunk_prediction = unit; // Does not ensure dictionary content consistency between endiannesses - *last_hash = hash; -} - -DENSITY_FORCE_INLINE void density_cheetah_decode_kernel_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const uint8_t flag, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) { - uint16_t hash; - uint32_t unit; - - switch (flag) { - case DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED: - density_cheetah_decode_process_predicted(out, last_hash, dictionary); - break; - case DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_A: - DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t)); - density_cheetah_decode_process_compressed_a(out, last_hash, dictionary, DENSITY_LITTLE_ENDIAN_16(hash)); - *in += sizeof(uint16_t); - break; - case DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_B: - DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t)); - density_cheetah_decode_process_compressed_b(out, last_hash, dictionary, DENSITY_LITTLE_ENDIAN_16(hash)); - *in += sizeof(uint16_t); - break; - default: // DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK - DENSITY_MEMCPY(&unit, *in, sizeof(uint32_t)); - density_cheetah_decode_process_uncompressed(out, last_hash, dictionary, unit); - *in += sizeof(uint32_t); - break; - } - - *out += sizeof(uint32_t); -} - -DENSITY_FORCE_INLINE void density_cheetah_decode_kernel_16(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const uint8_t flags, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) { - uint16_t hash; - uint32_t unit; - - switch (flags) { - DENSITY_CASE_GENERATOR_4_4_COMBINED(\ - density_cheetah_decode_process_predicted(out, last_hash, dictionary);, \ - DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED, \ - DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t)); \ - density_cheetah_decode_process_compressed_a(out, last_hash, dictionary, DENSITY_LITTLE_ENDIAN_16(hash));\ - *in += sizeof(uint16_t);, \ - DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_A, \ - DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t)); \ - density_cheetah_decode_process_compressed_b(out, last_hash, dictionary, DENSITY_LITTLE_ENDIAN_16(hash));\ - *in += sizeof(uint16_t);, \ - DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_B, \ - DENSITY_MEMCPY(&unit, *in, sizeof(uint32_t)); \ - density_cheetah_decode_process_uncompressed(out, last_hash, dictionary, unit);\ - *in += sizeof(uint32_t);, \ - DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK, \ - *out += sizeof(uint32_t);, \ - 2\ - ); - default: - break; - } - - *out += sizeof(uint32_t); -} - -DENSITY_FORCE_INLINE uint8_t density_cheetah_decode_read_flag(const density_cheetah_signature signature, const uint_fast8_t shift) { - return (uint8_t const) ((signature >> shift) & 0x3); -} - -DENSITY_FORCE_INLINE void density_cheetah_decode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const density_cheetah_signature signature, const uint_fast8_t shift, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) { - density_cheetah_decode_kernel_4(in, out, last_hash, density_cheetah_decode_read_flag(signature, shift), dictionary); -} - -DENSITY_FORCE_INLINE void density_cheetah_decode_16(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const density_cheetah_signature signature, const uint_fast8_t shift, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) { - density_cheetah_decode_kernel_16(in, out, last_hash, (uint8_t const) ((signature >> shift) & 0xff), dictionary); -} - -DENSITY_FORCE_INLINE void density_cheetah_decode_128(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const density_cheetah_signature signature, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) { -#ifdef __clang__ - uint_fast8_t count = 0; - for (uint_fast8_t count_b = 0; count_b < 8; count_b ++) { - density_cheetah_decode_16(in, out, last_hash, signature, count, dictionary); - count += 8; - } -#else - for (uint_fast8_t count_b = 0; count_b < density_bitsizeof(density_cheetah_signature); count_b += 8) - density_cheetah_decode_16(in, out, last_hash, signature, count_b, dictionary); -#endif -} - -DENSITY_FORCE_INLINE void density_cheetah_decode_read_signature(const uint8_t **DENSITY_RESTRICT in, density_cheetah_signature *DENSITY_RESTRICT signature) { -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(signature, *in, sizeof(density_cheetah_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - density_cheetah_signature endian_signature; - DENSITY_MEMCPY(&endian_signature, *in, sizeof(density_cheetah_signature)); - *signature = DENSITY_LITTLE_ENDIAN_64(endian_signature); -#else -#error -#endif - *in += sizeof(density_cheetah_signature); -} - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_cheetah_decode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) { - if (out_size < DENSITY_CHEETAH_DECOMPRESSED_UNIT_SIZE) - return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; - - density_cheetah_signature signature; - uint_fast8_t shift; - uint_fast64_t remaining; - uint_fast16_t last_hash = 0; - uint8_t flag; - - const uint8_t *start = *in; - - if (in_size < DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE) { - goto read_signature; - } - - const uint8_t *in_limit = *in + in_size - DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE; - uint8_t *out_limit = *out + out_size - DENSITY_CHEETAH_DECOMPRESSED_UNIT_SIZE; - - while (DENSITY_LIKELY(*in <= in_limit && *out <= out_limit)) { - if (DENSITY_UNLIKELY(!(state->counter & 0x1f))) { - DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START; - } - state->counter++; - if (DENSITY_UNLIKELY(state->copy_penalty)) { - DENSITY_ALGORITHM_COPY(DENSITY_CHEETAH_WORK_BLOCK_SIZE); - DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START; - } else { - const uint8_t *in_start = *in; - density_cheetah_decode_read_signature(in, &signature); - density_cheetah_decode_128(in, out, &last_hash, signature, (density_cheetah_dictionary *const) state->dictionary); - DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*in - in_start), DENSITY_CHEETAH_WORK_BLOCK_SIZE); - } - } - - if (*out > out_limit) - return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; - - read_signature: - if (in_size - (*in - start) < sizeof(density_cheetah_signature)) - return DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL; - shift = 0; - density_cheetah_decode_read_signature(in, &signature); - read_and_decode_4: - switch (in_size - (*in - start)) { - case 0: - case 1: - switch (density_cheetah_decode_read_flag(signature, shift)) { - case DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK: - goto process_remaining_bytes; // End marker - case DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED: - density_cheetah_decode_kernel_4(in, out, &last_hash, DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED, (density_cheetah_dictionary *const) state->dictionary); - shift += 2; - break; - default: - return DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING; - } - break; - case 2: - case 3: - flag = density_cheetah_decode_read_flag(signature, shift); - switch (flag) { - case DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK: - goto process_remaining_bytes; // End marker - default: - density_cheetah_decode_kernel_4(in, out, &last_hash, flag, (density_cheetah_dictionary *const) state->dictionary); - shift += 2; - break; - } - break; - default: - density_cheetah_decode_4(in, out, &last_hash, signature, shift, (density_cheetah_dictionary *const) state->dictionary); - shift += 2; - break; - } - - if (DENSITY_UNLIKELY(shift == density_bitsizeof(density_cheetah_signature))) - goto read_signature; - else - goto read_and_decode_4; - - process_remaining_bytes: - remaining = in_size - (*in - start); - DENSITY_ALGORITHM_COPY(remaining); - - return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED; -} diff --git a/contrib/density/src/algorithms/cheetah/core/cheetah_decode.h b/contrib/density/src/algorithms/cheetah/core/cheetah_decode.h deleted file mode 100644 index 278368e7938..00000000000 --- a/contrib/density/src/algorithms/cheetah/core/cheetah_decode.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 24/06/15 0:31 - * - * ----------------- - * Cheetah algorithm - * ----------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * Piotr Tarsa (https://github.com/tarsa) - * - * Description - * Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup - */ - -#ifndef DENSITY_CHEETAH_DECODE_H -#define DENSITY_CHEETAH_DECODE_H - -#include "../dictionary/cheetah_dictionary.h" -#include "../../algorithms.h" - -DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_cheetah_decode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t); - -#endif diff --git a/contrib/density/src/algorithms/cheetah/core/cheetah_encode.c b/contrib/density/src/algorithms/cheetah/core/cheetah_encode.c deleted file mode 100644 index 6cec16f4bf0..00000000000 --- a/contrib/density/src/algorithms/cheetah/core/cheetah_encode.c +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 23/06/15 23:29 - * - * ----------------- - * Cheetah algorithm - * ----------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * Piotr Tarsa (https://github.com/tarsa) - * - * Description - * Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup - */ - -#include "cheetah_encode.h" - -DENSITY_FORCE_INLINE void density_cheetah_encode_prepare_signature(uint8_t **DENSITY_RESTRICT out, density_cheetah_signature **DENSITY_RESTRICT signature_pointer, density_cheetah_signature *const DENSITY_RESTRICT signature) { - *signature = 0; - *signature_pointer = (density_cheetah_signature *) *out; - *out += sizeof(density_cheetah_signature); -} - -DENSITY_FORCE_INLINE void density_cheetah_encode_kernel(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const uint_fast16_t hash, const uint_fast8_t shift, density_cheetah_signature *const DENSITY_RESTRICT signature, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) { - uint32_t *predictedChunk = (uint32_t*) &dictionary->prediction_entries[*last_hash]; - - if (*predictedChunk ^ *unit) { - density_cheetah_dictionary_entry *found = &dictionary->entries[hash]; - uint32_t *found_a = &found->chunk_a; - if (*found_a ^ *unit) { - uint32_t *found_b = &found->chunk_b; - if (*found_b ^ *unit) { - *signature |= ((uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK << shift); - DENSITY_MEMCPY(*out, unit, sizeof(uint32_t)); - *out += sizeof(uint32_t); - } else { - *signature |= ((uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_B << shift); -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); -#elif defined(DENSITY_BIG_ENDIAN) - const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); - DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); -#else -#error -#endif - *out += sizeof(uint16_t); - } - *found_b = *found_a; - *found_a = *unit; // Does not ensure dictionary content consistency between endiannesses - } else { - *signature |= ((uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_A << shift); -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); -#elif defined(DENSITY_BIG_ENDIAN) - const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); - DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); -#else -#error -#endif - *out += sizeof(uint16_t); - } - *predictedChunk = *unit; // Does not ensure dictionary content consistency between endiannesses - } - *last_hash = hash; -} - -DENSITY_FORCE_INLINE void density_cheetah_encode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const uint_fast8_t shift, density_cheetah_signature *const DENSITY_RESTRICT signature, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) { - DENSITY_MEMCPY(unit, *in, sizeof(uint32_t)); - *in += sizeof(uint32_t); - density_cheetah_encode_kernel(out, last_hash, DENSITY_CHEETAH_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)), shift, signature, dictionary, unit); -} - -DENSITY_FORCE_INLINE void density_cheetah_encode_128(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_signature *const DENSITY_RESTRICT signature, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) { - uint_fast8_t count = 0; - -#ifdef __clang__ - for(; count < density_bitsizeof(density_cheetah_signature); count += 2) { - density_cheetah_encode_4(in, out, last_hash, count, signature, dictionary, unit); - } -#else - for (uint_fast8_t count_b = 0; count_b < 16; count_b++) { - DENSITY_UNROLL_2(\ - density_cheetah_encode_4(in, out, last_hash, count, signature, dictionary, unit);\ - count += 2); - } -#endif -} - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_cheetah_encode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) { - if (out_size < DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE) - return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; - - density_cheetah_signature signature; - density_cheetah_signature *signature_pointer; - uint_fast16_t last_hash = 0; - uint32_t unit; - - uint8_t *out_limit = *out + out_size - DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE; - uint_fast64_t limit_128 = (in_size >> 7); - - while (DENSITY_LIKELY(limit_128-- && *out <= out_limit)) { - if (DENSITY_UNLIKELY(!(state->counter & 0x1f))) { - DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START; - } - state->counter++; - if (DENSITY_UNLIKELY(state->copy_penalty)) { - DENSITY_ALGORITHM_COPY(DENSITY_CHEETAH_WORK_BLOCK_SIZE); - DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START; - } else { - const uint8_t *out_start = *out; - density_cheetah_encode_prepare_signature(out, &signature_pointer, &signature); - DENSITY_PREFETCH(*in + DENSITY_CHEETAH_WORK_BLOCK_SIZE); - density_cheetah_encode_128(in, out, &last_hash, &signature, (density_cheetah_dictionary *const) state->dictionary, &unit); -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_cheetah_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - const density_cheetah_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); - DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_cheetah_signature)); -#else -#error -#endif - DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*out - out_start), DENSITY_CHEETAH_WORK_BLOCK_SIZE); - } - } - - if (*out > out_limit) - return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; - - uint_fast64_t remaining; - - switch (in_size & 0x7f) { - case 0: - case 1: - case 2: - case 3: - density_cheetah_encode_prepare_signature(out, &signature_pointer, &signature); - signature = (uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK; // End marker -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_cheetah_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - const density_cheetah_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); - DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_cheetah_signature)); -#else -#error -#endif - goto process_remaining_bytes; - default: - break; - } - - const uint_fast64_t limit_4 = ((in_size & 0x7f) >> 2) << 1; // 4-byte units times number of signature flag bits - density_cheetah_encode_prepare_signature(out, &signature_pointer, &signature); - for (uint_fast8_t shift = 0; shift != limit_4; shift += 2) - density_cheetah_encode_4(in, out, &last_hash, shift, &signature, (density_cheetah_dictionary *const) state->dictionary, &unit); - - signature |= ((uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK << limit_4); // End marker -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_cheetah_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - const density_cheetah_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); - DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_cheetah_signature)); -#else -#error -#endif - - process_remaining_bytes: - remaining = in_size & 0x3; - if (remaining) - DENSITY_ALGORITHM_COPY(remaining); - - return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED; -} diff --git a/contrib/density/src/algorithms/cheetah/core/cheetah_encode.h b/contrib/density/src/algorithms/cheetah/core/cheetah_encode.h deleted file mode 100644 index 9f67e5bad9d..00000000000 --- a/contrib/density/src/algorithms/cheetah/core/cheetah_encode.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 23/06/15 23:29 - * - * ----------------- - * Cheetah algorithm - * ----------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * Piotr Tarsa (https://github.com/tarsa) - * - * Description - * Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup - */ - -#ifndef DENSITY_CHEETAH_ENCODE_H -#define DENSITY_CHEETAH_ENCODE_H - -#include "../dictionary/cheetah_dictionary.h" -#include "../../algorithms.h" - -DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_cheetah_encode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t); - -#endif diff --git a/contrib/density/src/algorithms/cheetah/dictionary/cheetah_dictionary.h b/contrib/density/src/algorithms/cheetah/dictionary/cheetah_dictionary.h deleted file mode 100644 index 5223d843574..00000000000 --- a/contrib/density/src/algorithms/cheetah/dictionary/cheetah_dictionary.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 06/12/13 20:20 - * - * ----------------- - * Cheetah algorithm - * ----------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * Piotr Tarsa (https://github.com/tarsa) - * - * Description - * Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup - */ - -#ifndef DENSITY_CHEETAH_DICTIONARY_H -#define DENSITY_CHEETAH_DICTIONARY_H - -#include "../cheetah.h" - -#include - -#pragma pack(push) -#pragma pack(4) -typedef struct { - uint32_t chunk_a; - uint32_t chunk_b; -} density_cheetah_dictionary_entry; - -typedef struct { - uint32_t next_chunk_prediction; -} density_cheetah_dictionary_prediction_entry; - -typedef struct { - density_cheetah_dictionary_entry entries[1 << DENSITY_CHEETAH_HASH_BITS]; - density_cheetah_dictionary_prediction_entry prediction_entries[1 << DENSITY_CHEETAH_HASH_BITS]; -} density_cheetah_dictionary; -#pragma pack(pop) - -#endif diff --git a/contrib/density/src/algorithms/dictionaries.c b/contrib/density/src/algorithms/dictionaries.c deleted file mode 100644 index 1e8d5466014..00000000000 --- a/contrib/density/src/algorithms/dictionaries.c +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Density - * - * Copyright (c) 2018, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 13/01/18 17:31 - */ - -#include "dictionaries.h" - -DENSITY_WINDOWS_EXPORT size_t density_get_dictionary_size(DENSITY_ALGORITHM algorithm) { - switch(algorithm) { - case DENSITY_ALGORITHM_CHAMELEON: - return sizeof(density_chameleon_dictionary); - case DENSITY_ALGORITHM_CHEETAH: - return sizeof(density_cheetah_dictionary); - case DENSITY_ALGORITHM_LION: - return sizeof(density_lion_dictionary); - default: - return 0; - } -} diff --git a/contrib/density/src/algorithms/dictionaries.h b/contrib/density/src/algorithms/dictionaries.h deleted file mode 100644 index 291c361963e..00000000000 --- a/contrib/density/src/algorithms/dictionaries.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Density - * - * Copyright (c) 2018, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 13/01/18 17:26 - */ - -#ifndef DENSITY_DICTIONARIES_H -#define DENSITY_DICTIONARIES_H - -#include "../globals.h" -#include "../algorithms/chameleon/dictionary/chameleon_dictionary.h" -#include "../algorithms/cheetah/dictionary/cheetah_dictionary.h" -#include "../algorithms/lion/dictionary/lion_dictionary.h" - -DENSITY_WINDOWS_EXPORT size_t density_get_dictionary_size(DENSITY_ALGORITHM); - -#endif diff --git a/contrib/density/src/algorithms/lion/core/lion_decode.c b/contrib/density/src/algorithms/lion/core/lion_decode.c deleted file mode 100644 index c84262c0d50..00000000000 --- a/contrib/density/src/algorithms/lion/core/lion_decode.c +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Density - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 24/06/15 20:55 - * - * -------------- - * Lion algorithm - * -------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Multiform compression algorithm - */ - -#include "lion_decode.h" - -DENSITY_FORCE_INLINE void density_lion_decode_read_signature(const uint8_t **DENSITY_RESTRICT in, uint_fast64_t *const DENSITY_RESTRICT signature) { -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(signature, *in, sizeof(density_lion_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - density_lion_signature endian_signature; - DENSITY_MEMCPY(&endian_signature, *in, sizeof(density_lion_signature)); - *signature = DENSITY_LITTLE_ENDIAN_64(endian_signature); -#else -#error -#endif - *in += sizeof(density_lion_signature); -} - -DENSITY_FORCE_INLINE void density_lion_decode_update_predictions_model(density_lion_dictionary_chunk_prediction_entry *const DENSITY_RESTRICT predictions, const uint32_t chunk) { - DENSITY_MEMMOVE((uint32_t *) predictions + 1, predictions, 2 * sizeof(uint32_t)); - *(uint32_t *) predictions = chunk; // Move chunk to the top of the predictions list, does not ensure dictionary content consistency between endiannesses -} - -DENSITY_FORCE_INLINE void density_lion_decode_update_dictionary_model(density_lion_dictionary_chunk_entry *const DENSITY_RESTRICT entry, const uint32_t chunk) { - DENSITY_MEMMOVE((uint32_t *) entry + 1, entry, 3 * sizeof(uint32_t)); - *(uint32_t *) entry = chunk; // Does not ensure dictionary content consistency between endiannesses -} - -DENSITY_FORCE_INLINE void density_lion_decode_read_hash(const uint8_t **DENSITY_RESTRICT in, uint16_t *DENSITY_RESTRICT const hash) { -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(hash, *in, sizeof(uint16_t)); -#elif defined(DENSITY_BIG_ENDIAN) - uint16_t endian_hash; - DENSITY_MEMCPY(&endian_hash, *in, sizeof(uint16_t)); - *hash = DENSITY_LITTLE_ENDIAN_16(endian_hash); -#else -#error -#endif - *in += sizeof(uint16_t); -} - -DENSITY_FORCE_INLINE void density_lion_decode_prediction_generic(uint8_t **DENSITY_RESTRICT out, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { - *hash = DENSITY_LION_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)); - DENSITY_MEMCPY(*out, unit, sizeof(uint32_t)); - *out += sizeof(uint32_t); -} - -DENSITY_FORCE_INLINE void density_lion_decode_dictionary_generic(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT const unit) { - DENSITY_MEMCPY(*out, unit, sizeof(uint32_t)); - *out += sizeof(uint32_t); - density_lion_dictionary_chunk_prediction_entry *prediction = &(dictionary->predictions[*last_hash]); - density_lion_decode_update_predictions_model(prediction, *unit); -} - -void density_lion_decode_prediction_a(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { - (void)in; - *unit = dictionary->predictions[*last_hash].next_chunk_a; - density_lion_decode_prediction_generic(out, hash, unit); - - *last_hash = *hash; -} - -void density_lion_decode_prediction_b(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { - (void)in; - density_lion_dictionary_chunk_prediction_entry *const prediction = &dictionary->predictions[*last_hash]; - *unit = prediction->next_chunk_b; - density_lion_decode_update_predictions_model(prediction, *unit); - density_lion_decode_prediction_generic(out, hash, unit); - - *last_hash = *hash; -} - -void density_lion_decode_prediction_c(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { - (void)in; - density_lion_dictionary_chunk_prediction_entry *const prediction = &dictionary->predictions[*last_hash]; - *unit = prediction->next_chunk_c; - density_lion_decode_update_predictions_model(prediction, *unit); - density_lion_decode_prediction_generic(out, hash, unit); - - *last_hash = *hash; -} - -void density_lion_decode_dictionary_a(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { - density_lion_decode_read_hash(in, hash); - DENSITY_PREFETCH(&dictionary->predictions[*hash]); - *unit = dictionary->chunks[*hash].chunk_a; - density_lion_decode_dictionary_generic(out, last_hash, dictionary, unit); - - *last_hash = *hash; -} - -void density_lion_decode_dictionary_b(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { - density_lion_decode_read_hash(in, hash); - DENSITY_PREFETCH(&dictionary->predictions[*hash]); - density_lion_dictionary_chunk_entry *entry = &dictionary->chunks[*hash]; - *unit = entry->chunk_b; - density_lion_decode_update_dictionary_model(entry, *unit); - density_lion_decode_dictionary_generic(out, last_hash, dictionary, unit); - - *last_hash = *hash; -} - -void density_lion_decode_dictionary_c(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { - density_lion_decode_read_hash(in, hash); - DENSITY_PREFETCH(&dictionary->predictions[*hash]); - density_lion_dictionary_chunk_entry *entry = &dictionary->chunks[*hash]; - *unit = entry->chunk_c; - density_lion_decode_update_dictionary_model(entry, *unit); - density_lion_decode_dictionary_generic(out, last_hash, dictionary, unit); - - *last_hash = *hash; -} - -void density_lion_decode_dictionary_d(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { - density_lion_decode_read_hash(in, hash); - DENSITY_PREFETCH(&dictionary->predictions[*hash]); - density_lion_dictionary_chunk_entry *entry = &dictionary->chunks[*hash]; - *unit = entry->chunk_d; - density_lion_decode_update_dictionary_model(entry, *unit); - density_lion_decode_dictionary_generic(out, last_hash, dictionary, unit); - - *last_hash = *hash; -} - -void density_lion_decode_plain(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) { - DENSITY_MEMCPY(unit, *in, sizeof(uint32_t)); - *in += sizeof(uint32_t); - *hash = DENSITY_LION_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)); - density_lion_dictionary_chunk_entry *entry = &dictionary->chunks[*hash]; - density_lion_decode_update_dictionary_model(entry, *unit); - DENSITY_MEMCPY(*out, unit, sizeof(uint32_t)); - *out += sizeof(uint32_t); - density_lion_dictionary_chunk_prediction_entry *prediction = &(dictionary->predictions[*last_hash]); - density_lion_decode_update_predictions_model(prediction, *unit); - - *last_hash = *hash; -} - -DENSITY_FORCE_INLINE void density_lion_decode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const data, const DENSITY_LION_FORM form) { - uint16_t hash; - uint32_t unit; - - data->attachments[form](in, out, last_hash, dictionary, &hash, &unit); -} - -DENSITY_FORCE_INLINE DENSITY_LION_FORM density_lion_decode_read_form(const uint8_t **DENSITY_RESTRICT in, uint_fast64_t *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_form_data *const form_data) { - const uint_fast8_t trailing_zeroes = DENSITY_CTZ(0x80 | (*signature >> *shift)); - if (DENSITY_LIKELY(!trailing_zeroes)) { - *shift = (uint_fast8_t)((*shift + 1) & 0x3f); - return density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool); - } else if (DENSITY_LIKELY(trailing_zeroes <= 6)) { - *shift = (uint_fast8_t)((*shift + (trailing_zeroes + 1)) & 0x3f); - return density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool + trailing_zeroes); - } else { - if (DENSITY_LIKELY(*shift <= (density_bitsizeof(density_lion_signature) - 7))) { - *shift = (uint_fast8_t)((*shift + 7) & 0x3f); - return density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool + 7); - } else { - density_lion_decode_read_signature(in, signature); - const uint_fast8_t primary_trailing_zeroes = (uint_fast8_t)(density_bitsizeof(density_lion_signature) - *shift); - const uint_fast8_t ctz_barrier_shift = (uint_fast8_t)(7 - primary_trailing_zeroes); - const uint_fast8_t secondary_trailing_zeroes = DENSITY_CTZ(((uint64_t)1 << ctz_barrier_shift) | *signature); - if (DENSITY_LIKELY(secondary_trailing_zeroes != ctz_barrier_shift)) - *shift = (uint_fast8_t)(secondary_trailing_zeroes + 1); - else - *shift = secondary_trailing_zeroes; - return density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool + primary_trailing_zeroes + secondary_trailing_zeroes); - } - } -} - -DENSITY_FORCE_INLINE void density_lion_decode_process_form(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const form_data, uint_fast64_t *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift) { - if (DENSITY_UNLIKELY(!*shift)) - density_lion_decode_read_signature(in, signature); - - switch ((*signature >> *shift) & 0x1) { - case 0: - density_lion_decode_4(in, out, last_hash, dictionary, form_data, density_lion_decode_read_form(in, signature, shift, form_data)); - break; - default: - density_lion_decode_4(in, out, last_hash, dictionary, form_data, density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool)); - *shift = (uint_fast8_t)((*shift + 1) & 0x3f); - break; - } -} - -DENSITY_FORCE_INLINE void density_lion_decode_256(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const form_data, uint_fast64_t *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift) { -#ifdef __clang__ - for (uint_fast8_t count = 0; count < (DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG >> 2); count++) { - DENSITY_UNROLL_4(density_lion_decode_process_form(in, out, last_hash, dictionary, form_data, signature, shift)); - } -#else - for (uint_fast8_t count = 0; count < (DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG >> 2); count++) { - DENSITY_UNROLL_4(density_lion_decode_process_form(in, out, last_hash, dictionary, form_data, signature, shift)); - } -#endif -} - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_lion_decode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) { - if (out_size < DENSITY_LION_MAXIMUM_DECOMPRESSED_UNIT_SIZE) - return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; - - density_lion_signature signature = 0; - density_lion_form_data data; - density_lion_form_model_init(&data); - void (*attachments[DENSITY_LION_NUMBER_OF_FORMS])(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const) = {(void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_prediction_a, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_prediction_b, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_prediction_c, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_dictionary_a, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_dictionary_b, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_dictionary_c, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_dictionary_d, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_plain}; - density_lion_form_model_attach(&data, attachments); - uint_fast8_t shift = 0; - uint_fast64_t remaining; - uint_fast16_t last_hash = 0; - DENSITY_LION_FORM form; - - const uint8_t *start = *in; - - if (in_size < DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE) { - goto read_and_decode_4; - } - - const uint8_t *in_limit = *in + in_size - DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE; - uint8_t *out_limit = *out + out_size - DENSITY_LION_MAXIMUM_DECOMPRESSED_UNIT_SIZE; - - while (DENSITY_LIKELY(*in <= in_limit && *out <= out_limit)) { - if (DENSITY_UNLIKELY(!(state->counter & 0xf))) { - DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START; - } - state->counter++; - if (DENSITY_UNLIKELY(state->copy_penalty)) { - DENSITY_ALGORITHM_COPY(DENSITY_LION_WORK_BLOCK_SIZE); - DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START; - } else { - const uint8_t *in_start = *in; - density_lion_decode_256(in, out, &last_hash, (density_lion_dictionary *const) state->dictionary, &data, &signature, &shift); - DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*in - in_start), DENSITY_LION_WORK_BLOCK_SIZE); - } - } - - if (*out > out_limit) - return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; - - read_and_decode_4: - if (DENSITY_UNLIKELY(!shift)) { - if (in_size - (*in - start) < sizeof(density_lion_signature)) - return DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL; - - density_lion_decode_read_signature(in, &signature); - } - form = density_lion_decode_read_form(in, &signature, &shift, &data); - switch (in_size - (*in - start)) { - case 0: - case 1: - switch (form) { - case DENSITY_LION_FORM_PLAIN: - goto process_remaining_bytes; // End marker - case DENSITY_LION_FORM_PREDICTIONS_A: - case DENSITY_LION_FORM_PREDICTIONS_B: - case DENSITY_LION_FORM_PREDICTIONS_C: - density_lion_decode_4(in, out, &last_hash, (density_lion_dictionary *const) state->dictionary, &data, form); - break; - default: - return DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING; // Not enough bytes to read a hash - } - break; - case 2: - case 3: - switch (form) { - case DENSITY_LION_FORM_PLAIN: - goto process_remaining_bytes; // End marker - default: - density_lion_decode_4(in, out, &last_hash, (density_lion_dictionary *const) state->dictionary, &data, form); - break; - } - break; - default: - density_lion_decode_4(in, out, &last_hash, (density_lion_dictionary *const) state->dictionary, &data, form); - break; - } - goto read_and_decode_4; - - process_remaining_bytes: - remaining = in_size - (*in - start); - DENSITY_MEMCPY(*out, *in, remaining); - *in += remaining; - *out += remaining; - - return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED; -} diff --git a/contrib/density/src/algorithms/lion/core/lion_decode.h b/contrib/density/src/algorithms/lion/core/lion_decode.h deleted file mode 100644 index 9972f0be388..00000000000 --- a/contrib/density/src/algorithms/lion/core/lion_decode.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Density - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 24/06/15 20:55 - * - * -------------- - * Lion algorithm - * -------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Multiform compression algorithm - */ - -#ifndef DENSITY_LION_DECODE_H -#define DENSITY_LION_DECODE_H - -#include "../dictionary/lion_dictionary.h" -#include "../forms/lion_form_model.h" -#include "../../algorithms.h" - -DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_lion_decode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t); - -#endif diff --git a/contrib/density/src/algorithms/lion/core/lion_encode.c b/contrib/density/src/algorithms/lion/core/lion_encode.c deleted file mode 100644 index 7cff2f3dda7..00000000000 --- a/contrib/density/src/algorithms/lion/core/lion_encode.c +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Density - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 24/06/15 18:57 - * - * -------------- - * Lion algorithm - * -------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Multiform compression algorithm - */ - -#include "lion_encode.h" - -DENSITY_FORCE_INLINE void density_lion_encode_prepare_signature(uint8_t **DENSITY_RESTRICT out, uint_fast64_t **DENSITY_RESTRICT signature_pointer, uint_fast64_t *const DENSITY_RESTRICT signature) { - *signature = 0; - *signature_pointer = (density_lion_signature *) *out; - *out += sizeof(density_lion_signature); -} - -DENSITY_FORCE_INLINE void density_lion_encode_push_to_proximity_signature(uint_fast64_t *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, const uint64_t content, const uint_fast8_t bits) { - *signature |= (content << *shift); - *shift += bits; -} - -DENSITY_FORCE_INLINE void density_lion_encode_push_to_signature(uint8_t **DENSITY_RESTRICT out, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, const uint64_t content, const uint_fast8_t bits) { - if (DENSITY_LIKELY(*shift)) { - density_lion_encode_push_to_proximity_signature(signature, shift, content, bits); - - if (DENSITY_UNLIKELY(*shift >= density_bitsizeof(density_lion_signature))) { -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(*signature_pointer, signature, sizeof(density_lion_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - const density_lion_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(*signature); - DENSITY_MEMCPY(*signature_pointer, &endian_signature, sizeof(density_lion_signature)); -#else -#error -#endif - - const uint_fast8_t remainder = (uint_fast8_t)(*shift & 0x3f); - *shift = 0; - if (remainder) { - density_lion_encode_prepare_signature(out, signature_pointer, signature); - density_lion_encode_push_to_proximity_signature(signature, shift, content >> (bits - remainder), remainder); - } - } - } else { - density_lion_encode_prepare_signature(out, signature_pointer, signature); - density_lion_encode_push_to_proximity_signature(signature, shift, content, bits); - } -} - -DENSITY_FORCE_INLINE void density_lion_encode_push_zero_to_signature(uint8_t **DENSITY_RESTRICT out, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, const uint_fast8_t bits) { - if (DENSITY_LIKELY(*shift)) { - *shift += bits; - - if (DENSITY_UNLIKELY(*shift >= density_bitsizeof(density_lion_signature))) { -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(*signature_pointer, signature, sizeof(density_lion_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - const density_lion_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(*signature); - DENSITY_MEMCPY(*signature_pointer, &endian_signature, sizeof(density_lion_signature)); -#else -#error -#endif - - const uint_fast8_t remainder = (uint_fast8_t)(*shift & 0x3f); - if (remainder) { - density_lion_encode_prepare_signature(out, signature_pointer, signature); - *shift = remainder; - } else - *shift = 0; - } - } else { - density_lion_encode_prepare_signature(out, signature_pointer, signature); - *shift = bits; - } -} - -DENSITY_FORCE_INLINE void density_lion_encode_push_code_to_signature(uint8_t **DENSITY_RESTRICT out, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, const density_lion_entropy_code code) { - density_lion_encode_push_to_signature(out, signature_pointer, signature, shift, code.value, code.bitLength); -} - -DENSITY_FORCE_INLINE void density_lion_encode_kernel_4(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, const uint16_t hash, density_lion_form_data *const data, const uint32_t unit) { - density_lion_dictionary_chunk_prediction_entry *const predictions = &dictionary->predictions[*last_hash]; - DENSITY_PREFETCH(&dictionary->predictions[hash]); - - if (*(uint32_t *) predictions ^ unit) { - if (*((uint32_t *) predictions + 1) ^ unit) { - if (*((uint32_t *) predictions + 2) ^ unit) { - density_lion_dictionary_chunk_entry *const in_dictionary = &dictionary->chunks[hash]; - if (*(uint32_t *) in_dictionary ^ unit) { - if (*((uint32_t *) in_dictionary + 1) ^ unit) { - if (*((uint32_t *) in_dictionary + 2) ^ unit) { - if (*((uint32_t *) in_dictionary + 3) ^ unit) { - density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_PLAIN)); - DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t)); - *out += sizeof(uint32_t); - } else { - density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_DICTIONARY_D)); -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); -#elif defined(DENSITY_BIG_ENDIAN) - const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); - DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); -#else -#error -#endif - *out += sizeof(uint16_t); - } - } else { - density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_DICTIONARY_C)); -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); -#elif defined(DENSITY_BIG_ENDIAN) - const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); - DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); -#else -#error -#endif - *out += sizeof(uint16_t); - } - } else { - density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_DICTIONARY_B)); -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); -#elif defined(DENSITY_BIG_ENDIAN) - const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); - DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); -#else -#error -#endif - *out += sizeof(uint16_t); - } - DENSITY_MEMMOVE((uint32_t *) in_dictionary + 1, in_dictionary, 3 * sizeof(uint32_t)); - *(uint32_t *) in_dictionary = unit; // Does not ensure dictionary content consistency between endiannesses - } else { - density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_DICTIONARY_A)); -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t)); -#elif defined(DENSITY_BIG_ENDIAN) - const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash); - DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t)); -#else -#error -#endif - *out += sizeof(uint16_t); - } - } else { - density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_PREDICTIONS_C)); - } - } else { - density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_PREDICTIONS_B)); - } - DENSITY_MEMMOVE((uint32_t *) predictions + 1, predictions, 2 * sizeof(uint32_t)); - *(uint32_t *) predictions = unit; // Does not ensure dictionary content consistency between endiannesses - } else - density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_PREDICTIONS_A)); - *last_hash = hash; -} - -DENSITY_FORCE_INLINE void density_lion_encode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const data, uint32_t *DENSITY_RESTRICT unit) { - DENSITY_MEMCPY(unit, *in, sizeof(uint32_t)); - density_lion_encode_kernel_4(out, last_hash, signature_pointer, signature, shift, dictionary, DENSITY_LION_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)), data, *unit); - *in += sizeof(uint32_t); -} - -DENSITY_FORCE_INLINE void density_lion_encode_generic(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, const uint_fast8_t chunks_per_process_unit, density_lion_form_data *const data, uint32_t *DENSITY_RESTRICT unit) { -#ifdef __clang__ - for (uint_fast8_t count = 0; count < (chunks_per_process_unit >> 2); count++) { - DENSITY_UNROLL_4(density_lion_encode_4(in, out, last_hash, signature_pointer, signature, shift, dictionary, data, unit)); - } -#else - for (uint_fast8_t count = 0; count < (chunks_per_process_unit >> 1); count++) { - DENSITY_UNROLL_2(density_lion_encode_4(in, out, last_hash, signature_pointer, signature, shift, dictionary, data, unit)); - } -#endif -} - -DENSITY_FORCE_INLINE void density_lion_encode_32(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const data, uint32_t *DENSITY_RESTRICT unit) { - density_lion_encode_generic(in, out, last_hash, signature_pointer, signature, shift, dictionary, DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_SMALL, data, unit); -} - -DENSITY_FORCE_INLINE void density_lion_encode_256(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const data, uint32_t *DENSITY_RESTRICT unit) { - density_lion_encode_generic(in, out, last_hash, signature_pointer, signature, shift, dictionary, DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG, data, unit); -} - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_lion_encode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) { - if (out_size < DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE) - return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; - - density_lion_signature signature = 0; - density_lion_signature *signature_pointer = NULL; - uint_fast8_t shift = 0; - density_lion_form_data data; - density_lion_form_model_init(&data); - uint_fast16_t last_hash = 0; - uint32_t unit; - - uint8_t *out_limit = *out + out_size - DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE; - uint_fast64_t limit_256 = (in_size >> 8); - - while (DENSITY_LIKELY(limit_256-- && *out <= out_limit)) { - if (DENSITY_UNLIKELY(!(state->counter & 0xf))) { - DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START; - } - state->counter++; - if (DENSITY_UNLIKELY(state->copy_penalty)) { - DENSITY_ALGORITHM_COPY(DENSITY_LION_WORK_BLOCK_SIZE); - DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START; - } else { - const uint8_t *out_start = *out; - DENSITY_PREFETCH(*in + DENSITY_LION_WORK_BLOCK_SIZE); - density_lion_encode_256(in, out, &last_hash, &signature_pointer, &signature, &shift, (density_lion_dictionary *const) state->dictionary, &data, &unit); - DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*out - out_start), DENSITY_LION_WORK_BLOCK_SIZE); - } - } - - if (*out > out_limit) - return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL; - - uint_fast64_t remaining; - - switch (in_size & 0xff) { - case 0: - case 1: - case 2: - case 3: - density_lion_encode_push_code_to_signature(out, &signature_pointer, &signature, &shift, density_lion_form_model_get_encoding(&data, DENSITY_LION_FORM_PLAIN)); // End marker -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_lion_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - const density_lion_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); - DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_lion_signature)); -#else -#error -#endif - goto process_remaining_bytes; - default: - break; - } - - uint_fast64_t limit_4 = (in_size & 0xff) >> 2; - while (limit_4--) - density_lion_encode_4(in, out, &last_hash, &signature_pointer, &signature, &shift, (density_lion_dictionary *const) state->dictionary, &data, &unit); - - density_lion_encode_push_code_to_signature(out, &signature_pointer, &signature, &shift, density_lion_form_model_get_encoding(&data, DENSITY_LION_FORM_PLAIN)); // End marker -#ifdef DENSITY_LITTLE_ENDIAN - DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_lion_signature)); -#elif defined(DENSITY_BIG_ENDIAN) - const density_lion_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature); - DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_lion_signature)); -#else -#error -#endif - - process_remaining_bytes: - remaining = in_size & 0x3; - if (remaining) { - DENSITY_MEMCPY(*out, *in, remaining); - *in += remaining; - *out += remaining; - } - - return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED; -} diff --git a/contrib/density/src/algorithms/lion/core/lion_encode.h b/contrib/density/src/algorithms/lion/core/lion_encode.h deleted file mode 100644 index 71cd2cad4cf..00000000000 --- a/contrib/density/src/algorithms/lion/core/lion_encode.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Density - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 24/06/15 18:56 - * - * -------------- - * Lion algorithm - * -------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Multiform compression algorithm - */ - -#ifndef DENSITY_LION_ENCODE_H -#define DENSITY_LION_ENCODE_H - -#include "../dictionary/lion_dictionary.h" -#include "../forms/lion_form_model.h" -#include "../../algorithms.h" - -DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_lion_encode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t); - -#endif diff --git a/contrib/density/src/algorithms/lion/dictionary/lion_dictionary.h b/contrib/density/src/algorithms/lion/dictionary/lion_dictionary.h deleted file mode 100644 index 68f71da6373..00000000000 --- a/contrib/density/src/algorithms/lion/dictionary/lion_dictionary.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Density - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 12/02/15 23:09 - * - * -------------- - * Lion algorithm - * -------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Multiform compression algorithm - */ - -#ifndef DENSITY_LION_DICTIONARY_H -#define DENSITY_LION_DICTIONARY_H - -#include "../lion.h" - -#pragma pack(push) -#pragma pack(4) - -typedef struct { - uint32_t chunk_a; - uint32_t chunk_b; - uint32_t chunk_c; - uint32_t chunk_d; - uint32_t chunk_e; -} density_lion_dictionary_chunk_entry; - -typedef struct { - uint32_t next_chunk_a; - uint32_t next_chunk_b; - uint32_t next_chunk_c; -} density_lion_dictionary_chunk_prediction_entry; - -typedef struct { - density_lion_dictionary_chunk_entry chunks[1 << DENSITY_LION_CHUNK_HASH_BITS]; - density_lion_dictionary_chunk_prediction_entry predictions[1 << DENSITY_LION_CHUNK_HASH_BITS]; -} density_lion_dictionary; -#pragma pack(pop) - -#endif diff --git a/contrib/density/src/algorithms/lion/forms/lion_form_model.c b/contrib/density/src/algorithms/lion/forms/lion_form_model.c deleted file mode 100644 index 21dfc23e3c9..00000000000 --- a/contrib/density/src/algorithms/lion/forms/lion_form_model.c +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Density - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 9/03/15 11:19 - * - * -------------- - * Lion algorithm - * -------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Multiform compression algorithm - */ - -#include "lion_form_model.h" - -const density_lion_entropy_code density_lion_form_entropy_codes[DENSITY_LION_NUMBER_OF_FORMS] = DENSITY_LION_FORM_MODEL_ENTROPY_CODES; - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_lion_form_model_init(density_lion_form_data *const data) { - density_lion_form_node *rank_0 = &data->formsPool[0]; - rank_0->form = DENSITY_LION_FORM_PLAIN; - rank_0->rank = 0; - rank_0->previousForm = NULL; - data->formsIndex[DENSITY_LION_FORM_PLAIN] = rank_0; - - density_lion_form_node *rank_1 = &data->formsPool[1]; - rank_1->form = DENSITY_LION_FORM_DICTIONARY_A; - rank_1->rank = 1; - rank_1->previousForm = rank_0; - data->formsIndex[DENSITY_LION_FORM_DICTIONARY_A] = rank_1; - - density_lion_form_node *rank_2 = &data->formsPool[2]; - rank_2->form = DENSITY_LION_FORM_DICTIONARY_B; - rank_2->rank = 2; - rank_2->previousForm = rank_1; - data->formsIndex[DENSITY_LION_FORM_DICTIONARY_B] = rank_2; - - density_lion_form_node *rank_3 = &data->formsPool[3]; - rank_3->form = DENSITY_LION_FORM_PREDICTIONS_A; - rank_3->rank = 3; - rank_3->previousForm = rank_2; - data->formsIndex[DENSITY_LION_FORM_PREDICTIONS_A] = rank_3; - - density_lion_form_node *rank_4 = &data->formsPool[4]; - rank_4->form = DENSITY_LION_FORM_PREDICTIONS_B; - rank_4->rank = 4; - rank_4->previousForm = rank_3; - data->formsIndex[DENSITY_LION_FORM_PREDICTIONS_B] = rank_4; - - density_lion_form_node *rank_5 = &data->formsPool[5]; - rank_5->form = DENSITY_LION_FORM_DICTIONARY_C; - rank_5->rank = 5; - rank_5->previousForm = rank_4; - data->formsIndex[DENSITY_LION_FORM_DICTIONARY_C] = rank_5; - - density_lion_form_node *rank_6 = &data->formsPool[6]; - rank_6->form = DENSITY_LION_FORM_PREDICTIONS_C; - rank_6->rank = 6; - rank_6->previousForm = rank_5; - data->formsIndex[DENSITY_LION_FORM_PREDICTIONS_C] = rank_6; - - density_lion_form_node *rank_7 = &data->formsPool[7]; - rank_7->form = DENSITY_LION_FORM_DICTIONARY_D; - rank_7->rank = 7; - rank_7->previousForm = rank_6; - data->formsIndex[DENSITY_LION_FORM_DICTIONARY_D] = rank_7; - - data->usages.usages_as_uint64_t = 0; -} - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_lion_form_model_attach(density_lion_form_data *const data, void (*attachments[DENSITY_LION_NUMBER_OF_FORMS])(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) { - for(uint_fast8_t count = 0; count < DENSITY_LION_NUMBER_OF_FORMS; count ++) - data->attachments[count] = attachments[count]; -} - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_lion_form_model_update(density_lion_form_data *const DENSITY_RESTRICT data, density_lion_form_node *const DENSITY_RESTRICT form, const uint8_t usage, density_lion_form_node *const DENSITY_RESTRICT previous_form, const uint8_t previous_usage) { - if (DENSITY_UNLIKELY(previous_usage < usage)) { // Relative stability is assumed - const DENSITY_LION_FORM form_value = form->form; - const DENSITY_LION_FORM previous_form_value = previous_form->form; - - previous_form->form = form_value; - form->form = previous_form_value; - - data->formsIndex[form_value] = previous_form; - data->formsIndex[previous_form_value] = form; - } -} - -DENSITY_FORCE_INLINE void density_lion_form_model_flatten(density_lion_form_data *const data, const uint8_t usage) { - if (DENSITY_UNLIKELY(usage & 0x80)) - data->usages.usages_as_uint64_t = (data->usages.usages_as_uint64_t >> 1) & 0x7f7f7f7f7f7f7f7fllu; // Flatten usage values -} - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE DENSITY_LION_FORM density_lion_form_model_increment_usage(density_lion_form_data *const data, density_lion_form_node *const DENSITY_RESTRICT form) { - const DENSITY_LION_FORM form_value = form->form; - const uint8_t usage = ++data->usages.usages_as_uint8_t[form_value]; - - density_lion_form_node *const previous_form = form->previousForm; - - if (previous_form) - density_lion_form_model_update(data, form, usage, previous_form, data->usages.usages_as_uint8_t[previous_form->form]); - else - density_lion_form_model_flatten(data, usage); - - return form_value; -} - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_lion_entropy_code density_lion_form_model_get_encoding(density_lion_form_data *const data, const DENSITY_LION_FORM form) { - const uint8_t usage = ++data->usages.usages_as_uint8_t[form]; - - density_lion_form_node *const form_found = data->formsIndex[form]; - density_lion_form_node *const previous_form = form_found->previousForm; - - if (previous_form) { - density_lion_form_model_update(data, form_found, usage, previous_form, data->usages.usages_as_uint8_t[previous_form->form]); - - return density_lion_form_entropy_codes[form_found->rank]; - } else { - density_lion_form_model_flatten(data, usage); - - return density_lion_form_entropy_codes[0]; - } -} diff --git a/contrib/density/src/algorithms/lion/forms/lion_form_model.h b/contrib/density/src/algorithms/lion/forms/lion_form_model.h deleted file mode 100644 index 90164d94c78..00000000000 --- a/contrib/density/src/algorithms/lion/forms/lion_form_model.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Density - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 9/03/15 12:04 - * - * -------------- - * Lion algorithm - * -------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Multiform compression algorithm - */ - -#ifndef DENSITY_LION_FORM_MODEL_H -#define DENSITY_LION_FORM_MODEL_H - -#include "../../../globals.h" -#include "../lion.h" - -#define DENSITY_LION_NUMBER_OF_FORMS 8 - -// Unary codes (reversed) except the last one -#define DENSITY_LION_FORM_MODEL_ENTROPY_CODES {\ - {DENSITY_BINARY_TO_UINT(1), 1},\ - {DENSITY_BINARY_TO_UINT(10), 2},\ - {DENSITY_BINARY_TO_UINT(100), 3},\ - {DENSITY_BINARY_TO_UINT(1000), 4},\ - {DENSITY_BINARY_TO_UINT(10000), 5},\ - {DENSITY_BINARY_TO_UINT(100000), 6},\ - {DENSITY_BINARY_TO_UINT(1000000), 7},\ - {DENSITY_BINARY_TO_UINT(0000000), 7},\ -} - -#pragma pack(push) -#pragma pack(4) -typedef struct { - void* previousForm; - DENSITY_LION_FORM form; - uint8_t rank; -} density_lion_form_node; - -typedef struct { - union { - uint8_t usages_as_uint8_t[DENSITY_LION_NUMBER_OF_FORMS]; - uint64_t usages_as_uint64_t; - } usages; - - void (*attachments[DENSITY_LION_NUMBER_OF_FORMS])(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const); - density_lion_form_node formsPool[DENSITY_LION_NUMBER_OF_FORMS]; - density_lion_form_node *formsIndex[DENSITY_LION_NUMBER_OF_FORMS]; - uint8_t nextAvailableForm; -} density_lion_form_data; -#pragma pack(pop) - -DENSITY_WINDOWS_EXPORT void density_lion_form_model_init(density_lion_form_data *const); - -DENSITY_WINDOWS_EXPORT void density_lion_form_model_attach(density_lion_form_data *const, void (*[DENSITY_LION_NUMBER_OF_FORMS])(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)); - -DENSITY_WINDOWS_EXPORT void density_lion_form_model_update(density_lion_form_data *const DENSITY_RESTRICT_DECLARE, density_lion_form_node *const DENSITY_RESTRICT_DECLARE, const uint8_t, density_lion_form_node *const DENSITY_RESTRICT_DECLARE, const uint8_t); - -DENSITY_WINDOWS_EXPORT DENSITY_LION_FORM density_lion_form_model_increment_usage(density_lion_form_data *const, density_lion_form_node *const DENSITY_RESTRICT_DECLARE); - -DENSITY_WINDOWS_EXPORT density_lion_entropy_code density_lion_form_model_get_encoding(density_lion_form_data *const, const DENSITY_LION_FORM); - -#endif diff --git a/contrib/density/src/algorithms/lion/lion.h b/contrib/density/src/algorithms/lion/lion.h deleted file mode 100644 index 30b40f8d92b..00000000000 --- a/contrib/density/src/algorithms/lion/lion.h +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Density - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 5/02/15 20:57 - * - * -------------- - * Lion algorithm - * -------------- - * - * Author(s) - * Guillaume Voirin (https://github.com/gpnuma) - * - * Description - * Multiform compression algorithm - */ - -#ifndef DENSITY_LION_H -#define DENSITY_LION_H - -#include "../../globals.h" - -#define DENSITY_LION_HASH32_MULTIPLIER (uint32_t)0x9D6EF916lu -#define DENSITY_LION_CHUNK_HASH_BITS 16 - -#define DENSITY_LION_HASH_ALGORITHM(value32) (uint16_t)(value32 * DENSITY_LION_HASH32_MULTIPLIER >> (32 - DENSITY_LION_CHUNK_HASH_BITS)) - -typedef enum { - DENSITY_LION_FORM_PREDICTIONS_A = 0, - DENSITY_LION_FORM_PREDICTIONS_B, - DENSITY_LION_FORM_PREDICTIONS_C, - DENSITY_LION_FORM_DICTIONARY_A, - DENSITY_LION_FORM_DICTIONARY_B, - DENSITY_LION_FORM_DICTIONARY_C, - DENSITY_LION_FORM_DICTIONARY_D, - DENSITY_LION_FORM_PLAIN, -} DENSITY_LION_FORM; - -typedef enum { - DENSITY_LION_PREDICTIONS_SIGNATURE_FLAG_A = 0x0, - DENSITY_LION_PREDICTIONS_SIGNATURE_FLAG_B = 0x1, -} DENSITY_LION_PREDICTIONS_SIGNATURE_FLAG; - -#pragma pack(push) -#pragma pack(4) -typedef struct { - uint_fast8_t value; - uint_fast8_t bitLength; -} density_lion_entropy_code; -#pragma pack(pop) - -typedef uint64_t density_lion_signature; - -#define DENSITY_LION_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE (density_bitsizeof(density_lion_signature) * sizeof(uint32_t)) // Plain writes -#define DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE (sizeof(density_lion_signature) + DENSITY_LION_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE) - -#define DENSITY_LION_MAXIMUM_DECOMPRESSED_UNIT_SIZE (density_bitsizeof(density_lion_signature) * sizeof(uint32_t)) // Smallest form size times work unit size - -#define DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_SMALL 8 -#define DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG 64 -#define DENSITY_LION_PROCESS_UNIT_SIZE_SMALL (DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_SMALL * sizeof(uint32_t)) -#define DENSITY_LION_PROCESS_UNIT_SIZE_BIG (DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG * sizeof(uint32_t)) - -#define DENSITY_LION_WORK_BLOCK_SIZE 256 -#define DENSITY_LION_COPY_PENALTY 2 - -#endif diff --git a/contrib/density/src/buffers/buffer.c b/contrib/density/src/buffers/buffer.c deleted file mode 100644 index d3e9943c599..00000000000 --- a/contrib/density/src/buffers/buffer.c +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Density - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 3/02/15 19:53 - */ - -#include "buffer.h" - -DENSITY_WINDOWS_EXPORT uint_fast64_t density_compress_safe_size(const uint_fast64_t input_size) { - const uint_fast64_t slack = DENSITY_MAX_3(DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE, DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE, DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE); - - // Chameleon longest output - uint_fast64_t chameleon_longest_output_size = 0; - chameleon_longest_output_size += sizeof(density_header); - chameleon_longest_output_size += sizeof(density_chameleon_signature) * (1 + (input_size >> (5 + 3))); // Signature space (1 bit <=> 4 bytes) - chameleon_longest_output_size += sizeof(density_chameleon_signature); // Eventual supplementary signature for end marker - chameleon_longest_output_size += input_size; // Everything encoded as plain data - - // Cheetah longest output - uint_fast64_t cheetah_longest_output_size = 0; - cheetah_longest_output_size += sizeof(density_header); - cheetah_longest_output_size += sizeof(density_cheetah_signature) * (1 + (input_size >> (4 + 3))); // Signature space (2 bits <=> 4 bytes) - cheetah_longest_output_size += sizeof(density_cheetah_signature); // Eventual supplementary signature for end marker - cheetah_longest_output_size += input_size; // Everything encoded as plain data - - // Lion longest output - uint_fast64_t lion_longest_output_size = 0; - lion_longest_output_size += sizeof(density_header); - lion_longest_output_size += sizeof(density_lion_signature) * (1 + ((input_size * 7) >> (5 + 3))); // Signature space (7 bits <=> 4 bytes), although this size is technically impossible - lion_longest_output_size += sizeof(density_lion_signature); // Eventual supplementary signature for end marker - lion_longest_output_size += input_size; // Everything encoded as plain data - - return DENSITY_MAX_3(chameleon_longest_output_size, cheetah_longest_output_size, lion_longest_output_size) + slack; -} - -DENSITY_WINDOWS_EXPORT uint_fast64_t density_decompress_safe_size(const uint_fast64_t expected_decompressed_output_size) { - const uint_fast64_t slack = DENSITY_MAX_3(DENSITY_CHAMELEON_DECOMPRESSED_UNIT_SIZE, DENSITY_CHEETAH_DECOMPRESSED_UNIT_SIZE, DENSITY_LION_MAXIMUM_DECOMPRESSED_UNIT_SIZE); - - return expected_decompressed_output_size + slack; -} - -DENSITY_FORCE_INLINE DENSITY_STATE density_convert_algorithm_exit_status(const density_algorithm_exit_status status) { - switch (status) { - case DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED: - return DENSITY_STATE_OK; - case DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL: - return DENSITY_STATE_ERROR_INPUT_BUFFER_TOO_SMALL; - case DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL: - return DENSITY_STATE_ERROR_OUTPUT_BUFFER_TOO_SMALL; - default: - return DENSITY_STATE_ERROR_DURING_PROCESSING; - } -} - -DENSITY_FORCE_INLINE density_processing_result density_make_result(const DENSITY_STATE state, const uint_fast64_t read, const uint_fast64_t written, density_context *const context) { - density_processing_result result; - result.state = state; - result.bytesRead = read; - result.bytesWritten = written; - result.context = context; - return result; -} - -DENSITY_FORCE_INLINE density_context* density_allocate_context(const DENSITY_ALGORITHM algorithm, const bool custom_dictionary, void *(*mem_alloc)(size_t)) { - density_context* context = mem_alloc(sizeof(density_context)); - context->algorithm = algorithm; - context->dictionary_size = density_get_dictionary_size(context->algorithm); - context->dictionary_type = custom_dictionary; - if(!context->dictionary_type) { - context->dictionary = mem_alloc(context->dictionary_size); - DENSITY_MEMSET(context->dictionary, 0, context->dictionary_size); - } - return context; -} - -DENSITY_WINDOWS_EXPORT void density_free_context(density_context *const context, void (*mem_free)(void *)) { - if(mem_free == NULL) - mem_free = free; - if(!context->dictionary_type) - mem_free(context->dictionary); - mem_free(context); -} - -DENSITY_WINDOWS_EXPORT density_processing_result density_compress_prepare_context(const DENSITY_ALGORITHM algorithm, const bool custom_dictionary, void *(*mem_alloc)(size_t)) { - if(mem_alloc == NULL) - mem_alloc = malloc; - - return density_make_result(DENSITY_STATE_OK, 0, 0, density_allocate_context(algorithm, custom_dictionary, mem_alloc)); -} - -DENSITY_WINDOWS_EXPORT density_processing_result density_compress_with_context(const uint8_t * input_buffer, const uint_fast64_t input_size, uint8_t * output_buffer, const uint_fast64_t output_size, density_context *const context) { - if (output_size < sizeof(density_header)) - return density_make_result(DENSITY_STATE_ERROR_OUTPUT_BUFFER_TOO_SMALL, 0, 0, context); - if(context == NULL) - return density_make_result(DENSITY_STATE_ERROR_INVALID_CONTEXT, 0, 0, context); - - // Variables setup - const uint8_t *in = input_buffer; - uint8_t *out = output_buffer; - density_algorithm_state state; - density_algorithm_exit_status status = DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING; - - // Header - density_header_write(&out, context->algorithm); - - // Compression - density_algorithms_prepare_state(&state, context->dictionary); - switch (context->algorithm) { - case DENSITY_ALGORITHM_CHAMELEON: - status = density_chameleon_encode(&state, &in, input_size, &out, output_size); - break; - case DENSITY_ALGORITHM_CHEETAH: - status = density_cheetah_encode(&state, &in, input_size, &out, output_size); - break; - case DENSITY_ALGORITHM_LION: - status = density_lion_encode(&state, &in, input_size, &out, output_size); - break; - } - - // Result - return density_make_result(density_convert_algorithm_exit_status(status), in - input_buffer, out - output_buffer, context); -} - -DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_prepare_context(const uint8_t *input_buffer, const uint_fast64_t input_size, const bool custom_dictionary, void *(*mem_alloc)(size_t)) { - if (input_size < sizeof(density_header)) - return density_make_result(DENSITY_STATE_ERROR_INPUT_BUFFER_TOO_SMALL, 0, 0, NULL); - - // Variables setup - const uint8_t* in = input_buffer; - if(mem_alloc == NULL) - mem_alloc = malloc; - - // Read header - density_header main_header; - density_header_read(&in, &main_header); - - // Setup context - density_context *const context = density_allocate_context(main_header.algorithm, custom_dictionary, mem_alloc); - return density_make_result(DENSITY_STATE_OK, in - input_buffer, 0, context); -} - -DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_with_context(const uint8_t * input_buffer, const uint_fast64_t input_size, uint8_t * output_buffer, const uint_fast64_t output_size, density_context *const context) { - if(context == NULL) - return density_make_result(DENSITY_STATE_ERROR_INVALID_CONTEXT, 0, 0, context); - - // Variables setup - const uint8_t *in = input_buffer; - uint8_t *out = output_buffer; - density_algorithm_state state; - density_algorithm_exit_status status = DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING; - - // Decompression - density_algorithms_prepare_state(&state, context->dictionary); - switch (context->algorithm) { - case DENSITY_ALGORITHM_CHAMELEON: - status = density_chameleon_decode(&state, &in, input_size, &out, output_size); - break; - case DENSITY_ALGORITHM_CHEETAH: - status = density_cheetah_decode(&state, &in, input_size, &out, output_size); - break; - case DENSITY_ALGORITHM_LION: - status = density_lion_decode(&state, &in, input_size, &out, output_size); - break; - } - - // Result - return density_make_result(density_convert_algorithm_exit_status(status), in - input_buffer, out - output_buffer, context); -} - -DENSITY_WINDOWS_EXPORT density_processing_result density_compress(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size, const DENSITY_ALGORITHM algorithm) { - density_processing_result result = density_compress_prepare_context(algorithm, false, malloc); - if(result.state) { - density_free_context(result.context, free); - return result; - } - - result = density_compress_with_context(input_buffer, input_size, output_buffer, output_size, result.context); - density_free_context(result.context, free); - return result; -} - -DENSITY_WINDOWS_EXPORT density_processing_result density_decompress(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size) { - density_processing_result result = density_decompress_prepare_context(input_buffer, input_size, false, malloc); - if(result.state) { - density_free_context(result.context, free); - return result; - } - - result = density_decompress_with_context(input_buffer + result.bytesRead, input_size - result.bytesRead, output_buffer, output_size, result.context); - density_free_context(result.context, free); - return result; -} diff --git a/contrib/density/src/buffers/buffer.h b/contrib/density/src/buffers/buffer.h deleted file mode 100644 index f34ab5458d2..00000000000 --- a/contrib/density/src/buffers/buffer.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Density - * - * Copyright (c) 2015, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 3/02/15 19:51 - */ - -#ifndef DENSITY_BUFFER_H -#define DENSITY_BUFFER_H - -#include "../globals.h" -#include "../density_api.h" -#include "../structure/header.h" -#include "../algorithms/chameleon/core/chameleon_encode.h" -#include "../algorithms/chameleon/core/chameleon_decode.h" -#include "../algorithms/cheetah/core/cheetah_encode.h" -#include "../algorithms/cheetah/core/cheetah_decode.h" -#include "../algorithms/lion/core/lion_encode.h" -#include "../algorithms/lion/core/lion_decode.h" - -DENSITY_WINDOWS_EXPORT uint_fast64_t density_compress_safe_size(const uint_fast64_t); -DENSITY_WINDOWS_EXPORT uint_fast64_t density_decompress_safe_size(const uint_fast64_t); -DENSITY_WINDOWS_EXPORT void density_free_context(density_context *const, void (*)(void *)); -DENSITY_WINDOWS_EXPORT density_processing_result density_compress_prepare_context(const DENSITY_ALGORITHM, const bool, void *(*)(size_t)); -DENSITY_WINDOWS_EXPORT density_processing_result density_compress_with_context(const uint8_t *, const uint_fast64_t, uint8_t *, const uint_fast64_t, density_context *const); -DENSITY_WINDOWS_EXPORT density_processing_result density_compress(const uint8_t *, const uint_fast64_t, uint8_t *, const uint_fast64_t, const DENSITY_ALGORITHM); -DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_prepare_context(const uint8_t *, const uint_fast64_t, const bool, void *(*)(size_t)); -DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_with_context(const uint8_t *, const uint_fast64_t, uint8_t *, const uint_fast64_t, density_context *const); -DENSITY_WINDOWS_EXPORT density_processing_result density_decompress(const uint8_t *, const uint_fast64_t, uint8_t *, const uint_fast64_t); - -#endif diff --git a/contrib/density/src/density_api.h b/contrib/density/src/density_api.h deleted file mode 100644 index 385e24e609a..00000000000 --- a/contrib/density/src/density_api.h +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 18/10/13 22:41 - */ - -#ifndef DENSITY_API_H -#define DENSITY_API_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -#if defined(_WIN64) || defined(_WIN32) -#define DENSITY_WINDOWS_EXPORT __declspec(dllexport) -#else -#define DENSITY_WINDOWS_EXPORT -#endif - - -/*********************************************************************************************************************** - * * - * API data structures * - * * - ***********************************************************************************************************************/ - -typedef uint8_t density_byte; -typedef bool density_bool; - -typedef enum { - DENSITY_ALGORITHM_CHAMELEON = 1, - DENSITY_ALGORITHM_CHEETAH = 2, - DENSITY_ALGORITHM_LION = 3, -} DENSITY_ALGORITHM; - -typedef enum { - DENSITY_STATE_OK = 0, // Everything went alright - DENSITY_STATE_ERROR_INPUT_BUFFER_TOO_SMALL, // Input buffer size is too small - DENSITY_STATE_ERROR_OUTPUT_BUFFER_TOO_SMALL, // Output buffer size is too small - DENSITY_STATE_ERROR_DURING_PROCESSING, // Error during processing - DENSITY_STATE_ERROR_INVALID_CONTEXT, // Invalid context - DENSITY_STATE_ERROR_INVALID_ALGORITHM, // Invalid algorithm -} DENSITY_STATE; - -typedef struct { - DENSITY_ALGORITHM algorithm; - bool dictionary_type; - size_t dictionary_size; - void* dictionary; -} density_context; - -typedef struct { - DENSITY_STATE state; - uint_fast64_t bytesRead; - uint_fast64_t bytesWritten; - density_context* context; -} density_processing_result; - - - -/*********************************************************************************************************************** - * * - * Density version information * - * * - ***********************************************************************************************************************/ - -/* - * Returns the major density version - */ -DENSITY_WINDOWS_EXPORT uint8_t density_version_major(void); - -/* - * Returns the minor density version - */ -DENSITY_WINDOWS_EXPORT uint8_t density_version_minor(void); - -/* - * Returns the density revision - */ -DENSITY_WINDOWS_EXPORT uint8_t density_version_revision(void); - - - -/*********************************************************************************************************************** - * * - * Density API functions * - * * - ***********************************************************************************************************************/ - -/* - * Return the required size of an algorithm's dictionary - * - * @param algorithm the algorithm to use this dictionary for - */ -DENSITY_WINDOWS_EXPORT size_t density_get_dictionary_size(DENSITY_ALGORITHM algorithm); - -/* - * Return an output buffer byte size which guarantees enough space for encoding input_size bytes - * - * @param input_size the size of the input data which is about to be compressed - */ -DENSITY_WINDOWS_EXPORT uint_fast64_t density_compress_safe_size(const uint_fast64_t input_size); - -/* - * Return an output buffer byte size which, if expected_decompressed_output_size is correct, will enable density to decompress properly - * - * @param expected_decompressed_output_size the expected (original) size of the decompressed data - */ -DENSITY_WINDOWS_EXPORT uint_fast64_t density_decompress_safe_size(const uint_fast64_t expected_decompressed_output_size); - -/* - * Releases a context from memory. - * - * @param context the context to free - * @param mem_free the memory freeing function. If set to NULL, free() is used - */ -DENSITY_WINDOWS_EXPORT void density_free_context(density_context *const context, void (*mem_free)(void *)); - -/* - * Allocate a context in memory using the provided function and optional dictionary - * - * @param algorithm the required algorithm - * @param custom_dictionary use an eventual custom dictionary ? If set to true the context's dictionary will have to be allocated - * @param mem_alloc the memory allocation function. If set to NULL, malloc() is used - */ -DENSITY_WINDOWS_EXPORT density_processing_result density_compress_prepare_context(const DENSITY_ALGORITHM algorithm, const bool custom_dictionary, void *(*mem_alloc)(size_t)); - -/* - * Compress an input_buffer of input_size bytes and store the result in output_buffer, using the provided context. - * Important note * this function could be unsafe memory-wise if not used properly. - * - * @param input_buffer a buffer of bytes - * @param input_size the size in bytes of input_buffer - * @param output_buffer a buffer of bytes - * @param output_size the size of output_buffer, must be at least DENSITY_MINIMUM_OUTPUT_BUFFER_SIZE - * @param context a pointer to a context structure - */ -DENSITY_WINDOWS_EXPORT density_processing_result density_compress_with_context(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size, density_context *const context); - -/* - * Compress an input_buffer of input_size bytes and store the result in output_buffer. - * - * @param input_buffer a buffer of bytes - * @param input_size the size in bytes of input_buffer - * @param output_buffer a buffer of bytes - * @param output_size the size of output_buffer, must be at least DENSITY_MINIMUM_OUTPUT_BUFFER_SIZE - * @param algorithm the algorithm to use - */ -DENSITY_WINDOWS_EXPORT density_processing_result density_compress(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size, const DENSITY_ALGORITHM algorithm); - -/* - * Reads the compressed data's header and creates an adequate decompression context. - * - * @param input_buffer a buffer of bytes - * @param input_size the size in bytes of input_buffer - * @param custom_dictionary use a custom dictionary ? If set to true the context's dictionary will have to be allocated - * @param mem_alloc the memory allocation function. If set to NULL, malloc() is used - */ -DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_prepare_context(const uint8_t *input_buffer, const uint_fast64_t input_size, const bool custom_dictionary, void *(*mem_alloc)(size_t)); - -/* - * Decompress an input_buffer of input_size bytes and store the result in output_buffer, using the provided dictionary. - * Important notes * You must know in advance the algorithm used for compression to provide the proper dictionary. - * * This function could be unsafe memory-wise if not used properly. - * - * @param input_buffer a buffer of bytes - * @param input_size the size in bytes of input_buffer - * @param output_buffer a buffer of bytes - * @param output_size the size of output_buffer, must be at least DENSITY_MINIMUM_OUTPUT_BUFFER_SIZE - * @param dictionaries a pointer to a dictionary - */ -DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_with_context(const uint8_t * input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size, density_context *const context); - -/* - * Decompress an input_buffer of input_size bytes and store the result in output_buffer. - * - * @param input_buffer a buffer of bytes - * @param input_size the size in bytes of input_buffer - * @param output_buffer a buffer of bytes - * @param output_size the size of output_buffer, must be at least DENSITY_MINIMUM_OUTPUT_BUFFER_SIZE - */ -DENSITY_WINDOWS_EXPORT density_processing_result density_decompress(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/contrib/density/src/globals.c b/contrib/density/src/globals.c deleted file mode 100644 index 4c58cd6f067..00000000000 --- a/contrib/density/src/globals.c +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 01/11/13 13:39 - */ - -#include "globals.h" - -DENSITY_WINDOWS_EXPORT uint8_t density_version_major() { - return DENSITY_MAJOR_VERSION; -} - -DENSITY_WINDOWS_EXPORT uint8_t density_version_minor() { - return DENSITY_MINOR_VERSION; -} - -DENSITY_WINDOWS_EXPORT uint8_t density_version_revision() { - return DENSITY_REVISION; -} diff --git a/contrib/density/src/globals.h b/contrib/density/src/globals.h deleted file mode 100644 index eb44a52a673..00000000000 --- a/contrib/density/src/globals.h +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 11/10/13 02:01 - */ - -#ifndef DENSITY_GLOBALS_H -#define DENSITY_GLOBALS_H - -#include -#include -#include -#include - -#include "density_api.h" - -#if defined(__clang__) || defined(__GNUC__) -#define DENSITY_FORCE_INLINE inline __attribute__((always_inline)) -#define DENSITY_RESTRICT restrict -#define DENSITY_RESTRICT_DECLARE -#define DENSITY_MEMCPY __builtin_memcpy -#define DENSITY_MEMMOVE __builtin_memmove -#define DENSITY_MEMSET __builtin_memset -#define DENSITY_LIKELY(x) __builtin_expect(!!(x), 1) -#define DENSITY_UNLIKELY(x) __builtin_expect(!!(x), 0) -#define DENSITY_PREFETCH(x) __builtin_prefetch(x) -#define DENSITY_CTZ(x) __builtin_ctz(x) - -#if defined(__BYTE_ORDER__) -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define DENSITY_LITTLE_ENDIAN -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -#define DENSITY_BIG_ENDIAN -#else -#error Unsupported endianness -#endif -#else -#error Unkwnown endianness -#endif - -#elif defined(_MSC_VER) -#include -#include - -#define DENSITY_FORCE_INLINE __forceinline -#define DENSITY_RESTRICT __restrict -#define DENSITY_RESTRICT_DECLARE __restrict -#define DENSITY_MEMCPY memcpy -#define DENSITY_MEMMOVE memmove -#define DENSITY_MEMSET memset -#define DENSITY_LIKELY(x) (x) -#define DENSITY_UNLIKELY(x) (x) -#define DENSITY_PREFETCH(x) ((void)(x)) - -DENSITY_FORCE_INLINE uint_fast8_t density_msvc_ctz(uint64_t value) { - unsigned long trailing_zero = 0; - if (_BitScanForward(&trailing_zero, (unsigned long)value)) - return (uint_fast8_t)trailing_zero; - else - return 0; -} -#define DENSITY_CTZ(x) density_msvc_ctz(x) - -#define DENSITY_LITTLE_ENDIAN // Little endian by default on Windows - -#else -#error Unsupported compiler -#endif - -#ifdef DENSITY_LITTLE_ENDIAN -#define DENSITY_LITTLE_ENDIAN_64(b) ((uint64_t)b) -#define DENSITY_LITTLE_ENDIAN_32(b) ((uint32_t)b) -#define DENSITY_LITTLE_ENDIAN_16(b) ((uint16_t)b) -#elif defined(DENSITY_BIG_ENDIAN) -#if __GNUC__ * 100 + __GNUC_MINOR__ >= 403 -#define DENSITY_LITTLE_ENDIAN_64(b) __builtin_bswap64(b) -#define DENSITY_LITTLE_ENDIAN_32(b) __builtin_bswap32(b) -#define DENSITY_LITTLE_ENDIAN_16(b) __builtin_bswap16(b) -#else -#warning Using bulk byte swap routines. Expect performance issues. -#define DENSITY_LITTLE_ENDIAN_64(b) ((((b) & 0xFF00000000000000ull) >> 56) | (((b) & 0x00FF000000000000ull) >> 40) | (((b) & 0x0000FF0000000000ull) >> 24) | (((b) & 0x000000FF00000000ull) >> 8) | (((b) & 0x00000000FF000000ull) << 8) | (((b) & 0x0000000000FF0000ull) << 24ull) | (((b) & 0x000000000000FF00ull) << 40) | (((b) & 0x00000000000000FFull) << 56)) -#define DENSITY_LITTLE_ENDIAN_32(b) ((((b) & 0xFF000000) >> 24) | (((b) & 0x00FF0000) >> 8) | (((b) & 0x0000FF00) << 8) | (((b) & 0x000000FF) << 24)) -#define DENSITY_LITTLE_ENDIAN_16(b) ((((b) & 0xFF00) >> 8) | (((b) & 0x00FF) << 8)) -#endif -#else -#error Unsupported endianness -#endif - -#define DENSITY_MAX_2(a, b) (((a)>(b))?(a):(b)) -#define DENSITY_MAX_3(a, b, c) (DENSITY_MAX_2(DENSITY_MAX_2(a, b), c)) - -#define DENSITY_FORMAT(v) 0##v##llu - -#define DENSITY_ISOLATE(b, p) ((DENSITY_FORMAT(b) / p) & 0x1) - -#define DENSITY_BINARY_TO_UINT(b) ((DENSITY_ISOLATE(b, 1llu) ? 0x1 : 0)\ - + (DENSITY_ISOLATE(b, 8llu) ? 0x2 : 0)\ - + (DENSITY_ISOLATE(b, 64llu) ? 0x4 : 0)\ - + (DENSITY_ISOLATE(b, 512llu) ? 0x8 : 0)\ - + (DENSITY_ISOLATE(b, 4096llu) ? 0x10 : 0)\ - + (DENSITY_ISOLATE(b, 32768llu) ? 0x20 : 0)\ - + (DENSITY_ISOLATE(b, 262144llu) ? 0x40 : 0)\ - + (DENSITY_ISOLATE(b, 2097152llu) ? 0x80 : 0)\ - + (DENSITY_ISOLATE(b, 16777216llu) ? 0x100 : 0)\ - + (DENSITY_ISOLATE(b, 134217728llu) ? 0x200 : 0)\ - + (DENSITY_ISOLATE(b, 1073741824llu) ? 0x400 : 0)\ - + (DENSITY_ISOLATE(b, 8589934592llu) ? 0x800 : 0)\ - + (DENSITY_ISOLATE(b, 68719476736llu) ? 0x1000 : 0)\ - + (DENSITY_ISOLATE(b, 549755813888llu) ? 0x2000 : 0)\ - + (DENSITY_ISOLATE(b, 4398046511104llu) ? 0x4000 : 0)\ - + (DENSITY_ISOLATE(b, 35184372088832llu) ? 0x8000 : 0)\ - + (DENSITY_ISOLATE(b, 281474976710656llu) ? 0x10000 : 0)\ - + (DENSITY_ISOLATE(b, 2251799813685248llu) ? 0x20000 : 0)) - -#define DENSITY_UNROLL_2(op) op; op -#define DENSITY_UNROLL_4(op) DENSITY_UNROLL_2(op); DENSITY_UNROLL_2(op) -#define DENSITY_UNROLL_8(op) DENSITY_UNROLL_4(op); DENSITY_UNROLL_4(op) -#define DENSITY_UNROLL_16(op) DENSITY_UNROLL_8(op); DENSITY_UNROLL_8(op) -#define DENSITY_UNROLL_32(op) DENSITY_UNROLL_16(op); DENSITY_UNROLL_16(op) -#define DENSITY_UNROLL_64(op) DENSITY_UNROLL_32(op); DENSITY_UNROLL_32(op) - -#define DENSITY_CASE_GENERATOR_2(op_a, flag_a, op_b, flag_b, op_mid, shift)\ - case ((flag_b << shift) | flag_a):\ - op_a;\ - op_mid;\ - op_b;\ - break; - -#define DENSITY_CASE_GENERATOR_4(op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ - case ((flag_d << (shift * 3)) | (flag_c << (shift * 2)) | (flag_b << shift) | flag_a):\ - op_a;\ - op_mid;\ - op_b;\ - op_mid;\ - op_c;\ - op_mid;\ - op_d;\ - break; - -#define DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_1, flag_1, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ - DENSITY_CASE_GENERATOR_2(op_1, flag_1, op_a, flag_a, op_mid, shift);\ - DENSITY_CASE_GENERATOR_2(op_1, flag_1, op_b, flag_b, op_mid, shift);\ - DENSITY_CASE_GENERATOR_2(op_1, flag_1, op_c, flag_c, op_mid, shift);\ - DENSITY_CASE_GENERATOR_2(op_1, flag_1, op_d, flag_d, op_mid, shift); - -#define DENSITY_CASE_GENERATOR_4_2_COMBINED(op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ - DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_a, flag_a, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_b, flag_b, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_c, flag_c, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_d, flag_d, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift); - -#define DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ - DENSITY_CASE_GENERATOR_4(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_a, flag_a, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_b, flag_b, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_c, flag_c, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_d, flag_d, op_mid, shift); - -#define DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_2, flag_2, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ - DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_a, flag_a, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_b, flag_b, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_c, flag_c, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_d, flag_d, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift); - -#define DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_1, flag_1, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ - DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_a, flag_a, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_b, flag_b, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_c, flag_c, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_d, flag_d, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift); - -#define DENSITY_CASE_GENERATOR_4_4_COMBINED(op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\ - DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_a, flag_a, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_b, flag_b, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_c, flag_c, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\ - DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_d, flag_d, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift); - -#define DENSITY_DICTIONARY_PREFERRED_RESET_CYCLE_SHIFT 6 -#define DENSITY_DICTIONARY_PREFERRED_RESET_CYCLE (1 << DENSITY_DICTIONARY_PREFERRED_RESET_CYCLE_SHIFT) - - -#define density_bitsizeof(x) (8 * sizeof(x)) - -#define DENSITY_SPOOKYHASH_SEED_1 (0xabc) -#define DENSITY_SPOOKYHASH_SEED_2 (0xdef) - -DENSITY_WINDOWS_EXPORT uint8_t density_version_major(); - -DENSITY_WINDOWS_EXPORT uint8_t density_version_minor(); - -DENSITY_WINDOWS_EXPORT uint8_t density_version_revision(); - - -/********************************************************************************************************************** - * * - * Global compile-time switches * - * * - **********************************************************************************************************************/ - -#define DENSITY_MAJOR_VERSION 0 -#define DENSITY_MINOR_VERSION 14 -#define DENSITY_REVISION 2 - - - -#endif diff --git a/contrib/density/src/structure/header.c b/contrib/density/src/structure/header.c deleted file mode 100644 index 4306a5962cf..00000000000 --- a/contrib/density/src/structure/header.c +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 11/10/13 17:56 - */ - -#include "header.h" - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_header_read(const uint8_t **DENSITY_RESTRICT in, density_header *DENSITY_RESTRICT header) { - header->version[0] = *(*in); - header->version[1] = *(*in + 1); - header->version[2] = *(*in + 2); - header->algorithm = *(*in + 3); - - *in += sizeof(density_header); -} - -DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_header_write(uint8_t **DENSITY_RESTRICT out, const DENSITY_ALGORITHM algorithm) { - *(*out) = DENSITY_MAJOR_VERSION; - *(*out + 1) = DENSITY_MINOR_VERSION; - *(*out + 2) = DENSITY_REVISION; - *(*out + 3) = algorithm; - *(*out + 4) = 0; - *(*out + 5) = 0; - *(*out + 6) = 0; - *(*out + 7) = 0; - - *out += sizeof(density_header); -} diff --git a/contrib/density/src/structure/header.h b/contrib/density/src/structure/header.h deleted file mode 100644 index d4065b5600c..00000000000 --- a/contrib/density/src/structure/header.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Density - * - * Copyright (c) 2013, Guillaume Voirin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * 11/10/13 02:06 - */ - -#ifndef DENSITY_FILE_HEADER_H -#define DENSITY_FILE_HEADER_H - -#include -#include - -#include "../globals.h" -#include "../density_api.h" - -#pragma pack(push) -#pragma pack(4) - -typedef struct { - density_byte version[3]; - density_byte algorithm; - density_byte reserved[4]; -} density_header; - -#pragma pack(pop) - -DENSITY_WINDOWS_EXPORT void density_header_read(const uint8_t ** DENSITY_RESTRICT_DECLARE, density_header * DENSITY_RESTRICT_DECLARE); -DENSITY_WINDOWS_EXPORT void density_header_write(uint8_t ** DENSITY_RESTRICT_DECLARE, const DENSITY_ALGORITHM); - -#endif diff --git a/contrib/lizard/.gitattributes b/contrib/lizard/.gitattributes deleted file mode 100644 index 6212bd405b4..00000000000 --- a/contrib/lizard/.gitattributes +++ /dev/null @@ -1,21 +0,0 @@ -# Set the default behavior -* text eol=lf - -# Explicitly declare source files -*.c text eol=lf -*.h text eol=lf - -# Denote files that should not be modified. -*.odt binary -*.png binary - -# Visual Studio -*.sln text eol=crlf -*.vcxproj* text eol=crlf -*.vcproj* text eol=crlf -*.suo binary -*.rc text eol=crlf - -# Windows -*.bat text eol=crlf -*.cmd text eol=crlf diff --git a/contrib/lizard/.gitignore b/contrib/lizard/.gitignore deleted file mode 100644 index dc30b0e5504..00000000000 --- a/contrib/lizard/.gitignore +++ /dev/null @@ -1,37 +0,0 @@ -# Object files -*.o -*.ko - -# Libraries -*.lib -*.a - -# Shared objects (inc. Windows DLLs) -*.dll -*.so -*.so.* -*.dylib - -# Executables -*.exe -*.out -*.app - -# IDE / editors files -*.suo -*.user -.clang_complete - -# Directories -_lizardbench/ -_visual/ -_codelite/ -_backup/ -_codelite_lz4/ - -# Archives -*.zip -*.liz - -*.txt -*.bat \ No newline at end of file diff --git a/contrib/lizard/.travis.yml b/contrib/lizard/.travis.yml deleted file mode 100644 index 29f75591a31..00000000000 --- a/contrib/lizard/.travis.yml +++ /dev/null @@ -1,268 +0,0 @@ -language: c -matrix: - fast_finish: true - include: - - name: macOS - os: osx - env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lizard CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang - - - # Ubuntu 18.04 - - name: Ubuntu 18.04, gcc-9, 64-bit and 32-bit tests - dist: bionic - script: - - CC=gcc-9; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - - CC=gcc-9; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror - before_install: - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get -q update - - sudo apt-get --no-install-suggests --no-install-recommends -yq install gcc-multilib gcc-9 gcc-9-multilib - - - name: Ubuntu 18.04, clang-9, 64-bit tests - dist: bionic - script: - - CC=clang-9; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - sourceline: 'deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main' - key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' - packages: - - clang-9 - - - - # Ubuntu 16.04 gcc - - name: Ubuntu 16.04, gcc-9, 64-bit and 32-bit tests - dist: xenial - script: - - CC=gcc-9; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - - CC=gcc-9; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-multilib - - gcc-9 - - gcc-9-multilib - - - name: Ubuntu 16.04, gcc-8, 64-bit and 32-bit tests - dist: xenial - script: - - CC=gcc-8; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - - CC=gcc-8; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-multilib - - gcc-8 - - gcc-8-multilib - - - name: Ubuntu 16.04, gcc-7, 64-bit and 32-bit tests - dist: xenial - script: - - CC=gcc-7; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - - CC=gcc-7; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-multilib - - gcc-7 - - gcc-7-multilib - - - name: Ubuntu 16.04, gcc-6, 64-bit and 32-bit tests - dist: xenial - script: - - CC=gcc-6; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - - CC=gcc-6; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-multilib - - gcc-6 - - gcc-6-multilib - - - name: Ubuntu 16.04, gcc-5, 64-bit and 32-bit tests - dist: xenial - script: - - CC=gcc-5; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - - CC=gcc-5; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-multilib - - gcc-5 - - gcc-5-multilib - - - name: Ubuntu 16.04, gcc-4.8, 64-bit and 32-bit tests - dist: xenial - script: - - CC=gcc-4.8; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - - CC=gcc-4.8; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror - addons: - apt: - packages: - - gcc-multilib - - gcc-4.8 - - gcc-4.8-multilib - - - - # Ubuntu 16.04 clang - - name: Ubuntu 16.04, clang-9, 64-bit tests - dist: xenial - script: - - CC=clang-9; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - sourceline: 'deb https://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main' - key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' - packages: - - clang-9 - - - name: Ubuntu 16.04, clang-8, 64-bit tests - dist: xenial - script: - - CC=clang-8; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-xenial-8 - packages: - - clang-8 - - - name: Ubuntu 16.04, clang-7, 64-bit tests - dist: xenial - script: - - CC=clang-7; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-xenial-7 - packages: - - clang-7 - - - name: Ubuntu 16.04, clang-6.0, 64-bit tests - dist: xenial - script: - - CC=clang-6.0; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-xenial-6.0 - packages: - - clang-6.0 - - - name: Ubuntu 16.04, clang-5.0, 64-bit tests - dist: xenial - script: - - CC=clang-5.0; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-xenial-5.0 - packages: - - clang-5.0 - - - name: Ubuntu 16.04, clang-4.0, 64-bit tests - dist: xenial - script: - - CC=clang-4.0; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-xenial-4.0 - packages: - - clang-4.0 - - - name: Ubuntu 14.04, clang-3.8, 64-bit tests - dist: trusty - script: - - CC=clang-3.8; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-trusty-3.8 - packages: - - clang-3.8 - - - - - name: Ubuntu 16.04, misc tests - env: Cmd="make c_standards && make gpptest-native && make clean && make examples && make clean && make cmake && make clean && make travis-install && make clean && make clangtest-native" COMPILER=cc - dist: xenial - - - name: Ubuntu 14.04, arm and aarch64 - env: Cmd='make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static && make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static' COMPILER=arm-linux-gnueabi-gcc - dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-arm - - qemu-user-static - - gcc-arm-linux-gnueabi - - libc6-dev-armel-cross - - gcc-aarch64-linux-gnu - - libc6-dev-arm64-cross - - - name: Ubuntu 14.04, powerpc and ppc64 - env: Cmd='make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64' COMPILER=powerpc-linux-gnu-gcc - dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-ppc - - qemu-user-static - - gcc-powerpc-linux-gnu - - - name: Ubuntu 14.04, valgrind - env: Cmd='make -C tests test-mem MOREFLAGS=-DLIZARD_RESET_MEM' COMPILER=cc - dist: trusty - sudo: required - addons: - apt: - packages: - - valgrind - - - #- env: Ubu=14.04 Cmd='make sanitize' COMPILER=clang - # dist: trusty - # sudo: required - # addons: - # apt: - # packages: - # - valgrind - - #- env: Ubu=14.04 Cmd='make staticAnalyze' COMPILER=clang - # dist: trusty - # sudo: required - # addons: - # apt: - # packages: - # - clang - - -script: - - echo Cmd=$Cmd - - $COMPILER -v - - sh -c "$Cmd" diff --git a/contrib/lizard/LICENSE b/contrib/lizard/LICENSE deleted file mode 100644 index df1edbca85e..00000000000 --- a/contrib/lizard/LICENSE +++ /dev/null @@ -1,15 +0,0 @@ -This repository uses 2 different licenses : -- all files in the `lib` directory use a BSD 2-Clause license -- all other files use a GPLv2 license, unless explicitly stated otherwise - -Relevant license is reminded at the top of each source file, -and with the presence of COPYING or LICENSE file. - -This model emphasizes the fact that -only files in the `lib` directory are designed to be included into 3rd party projects. - -Other files, such as those from `programs` or `examples` directory, -are not intended to be compiled outside of their context. -They can serve as source of inspiration, -but they should not be copy/pasted into 3rd party projects, -as this scenario is not supported. diff --git a/contrib/lizard/NEWS b/contrib/lizard/NEWS deleted file mode 100644 index 05d424656e9..00000000000 --- a/contrib/lizard/NEWS +++ /dev/null @@ -1,41 +0,0 @@ -Lizard v1.0 -- LZ5 v2.0 was renamed to Lizard v1.0 -- improved compression speed with many small files - -LZ5 v2.0 -- this version is optimized for decompression speed (instead of ratio for previous versions) -- LZ5 v2.0 contains 4 compression methods: - fastLZ4 : compression levels -10...-19 are designed to give better decompression speed than LZ4 i.e. over 2000 MB/s - LIZv1 : compression levels -20...-29 are designed to give better ratio than LZ4 keeping 75% decompression speed - fastLZ4 + Huffman : compression levels -30...-39 add Huffman coding to fastLZ4 - LIZv1 + Huffman : compression levels -40...-49 give the best ratio (comparable to zlib and low levels of zstd/brotli) at decompression speed of 1000 MB/s - -LZ5 v1.5 -- introduced compatibility with Visual C++ 2010 and newer -- attached Visual Studio 2010 project -- thoroughly tested with 21 Travis CI and 7 AppVeyor CI tests -- fixed bug with reusing a context in lizard_frame.c (LizardF_compressBegin and Lizard_compress_HC_continue) -- fixed rare bug in match finder (concerns levels 4 - 15) - -LZ5 v1.4.1 -- fixed bug with a backward match extension (level 11 and 12) - -LZ5 v1.4 -- improved: levels from 13 to 15 (maximum compression ratio) -- added a new parser: LizardHC_optimal_price_bt -- updated documentation: lizard_Block_format.md and lizard_Frame_format.md -- changed lizard.exe: the "-B" option with block size [1-7] = 64KB, 256KB, 1MB, 4MB, 16MB, 64MB, 256MB (default : 4 = 4MB) - -LZ5 v1.3.3 -- added: new levels from 11 to 18 (maximum compression ratio) -- added: a new parser: LizardHC_optimal_price -- fixed: buffer-overflow during decompression (thanks to m^2) - -LZ5 r132 -- improved compression ratio -- added: new parsers: LizardHC_fast, LizardHC_price_fast, LizardHC_lowest_price -- added: a special 1-byte codeword for the last occured offset -- added: support for 3-byte long matches (MINMATCH = 3) - -LZ5 r131 -The first release based on LZ4 r132 dev diff --git a/contrib/lizard/README.md b/contrib/lizard/README.md deleted file mode 100644 index 50dd737fb8d..00000000000 --- a/contrib/lizard/README.md +++ /dev/null @@ -1,95 +0,0 @@ -Lizard - efficient compression with very fast decompression --------------------------------------------------------- - -Lizard (formerly LZ5) is a lossless compression algorithm which contains 4 compression methods: -- fastLZ4 : compression levels -10...-19 are designed to give better decompression speed than [LZ4] i.e. over 2000 MB/s -- LIZv1 : compression levels -20...-29 are designed to give better ratio than [LZ4] keeping 75% decompression speed -- fastLZ4 + Huffman : compression levels -30...-39 add Huffman coding to fastLZ4 -- LIZv1 + Huffman : compression levels -40...-49 give the best ratio (comparable to [zlib] and low levels of [zstd]/[brotli]) at decompression speed of 1000 MB/s - -Lizard library is based on frequently used [LZ4] library by Yann Collet but the Lizard compression format is not compatible with LZ4. -Lizard library is provided as open-source software using BSD 2-Clause license. -The high compression/decompression speed is achieved without any SSE and AVX extensions. - - -|Branch |Status | -|------------|---------| -|lz5_v1.5 | [![Build Status][travis15Badge]][travisLink] [![Build status][Appveyor15Badge]][AppveyorLink] | -|lizard | [![Build Status][travis20Badge]][travisLink] [![Build status][Appveyor20Badge]][AppveyorLink] | - -[travis15Badge]: https://travis-ci.org/inikep/lizard.svg?branch=lz5_v1.5 "Continuous Integration test suite" -[travis20Badge]: https://travis-ci.org/inikep/lizard.svg?branch=lizard "Continuous Integration test suite" -[travisLink]: https://travis-ci.org/inikep/lizard -[Appveyor15Badge]: https://ci.appveyor.com/api/projects/status/cqw7emcuqge369p0/branch/lz5_v1.5?svg=true "Visual test suite" -[Appveyor20Badge]: https://ci.appveyor.com/api/projects/status/cqw7emcuqge369p0/branch/lizard?svg=true "Visual test suite" -[AppveyorLink]: https://ci.appveyor.com/project/inikep/lizard -[LZ4]: https://github.com/lz4/lz4 -[zlib]: https://github.com/madler/zlib -[zstd]: https://github.com/facebook/zstd -[brotli]: https://github.com/google/brotli - - -Benchmarks -------------------------- - -The following results are obtained with [lzbench](https://github.com/inikep/lzbench) and `-t16,16` -using 1 core of Intel Core i5-4300U, Windows 10 64-bit (MinGW-w64 compilation under gcc 6.2.0) -with [silesia.tar] which contains tarred files from [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia). - -| Compressor name | Compression| Decompress.| Compr. size | Ratio | -| --------------- | -----------| -----------| ----------- | ----- | -| memcpy | 7332 MB/s | 8719 MB/s | 211947520 |100.00 | -| lz4 1.7.3 | 440 MB/s | 2318 MB/s | 100880800 | 47.60 | -| lz4hc 1.7.3 -1 | 98 MB/s | 2121 MB/s | 87591763 | 41.33 | -| lz4hc 1.7.3 -4 | 55 MB/s | 2259 MB/s | 79807909 | 37.65 | -| lz4hc 1.7.3 -9 | 22 MB/s | 2315 MB/s | 77892285 | 36.75 | -| lz4hc 1.7.3 -12 | 17 MB/s | 2323 MB/s | 77849762 | 36.73 | -| lz4hc 1.7.3 -16 | 10 MB/s | 2323 MB/s | 77841782 | 36.73 | -| lizard 1.0 -10 | 346 MB/s | 2610 MB/s | 103402971 | 48.79 | -| lizard 1.0 -12 | 103 MB/s | 2458 MB/s | 86232422 | 40.69 | -| lizard 1.0 -15 | 50 MB/s | 2552 MB/s | 81187330 | 38.31 | -| lizard 1.0 -19 | 3.04 MB/s | 2497 MB/s | 77416400 | 36.53 | -| lizard 1.0 -21 | 157 MB/s | 1795 MB/s | 89239174 | 42.10 | -| lizard 1.0 -23 | 30 MB/s | 1778 MB/s | 81097176 | 38.26 | -| lizard 1.0 -26 | 6.63 MB/s | 1734 MB/s | 74503695 | 35.15 | -| lizard 1.0 -29 | 1.37 MB/s | 1634 MB/s | 68694227 | 32.41 | -| lizard 1.0 -30 | 246 MB/s | 909 MB/s | 85727429 | 40.45 | -| lizard 1.0 -32 | 94 MB/s | 1244 MB/s | 76929454 | 36.30 | -| lizard 1.0 -35 | 47 MB/s | 1435 MB/s | 73850400 | 34.84 | -| lizard 1.0 -39 | 2.94 MB/s | 1502 MB/s | 69807522 | 32.94 | -| lizard 1.0 -41 | 126 MB/s | 961 MB/s | 76100661 | 35.91 | -| lizard 1.0 -43 | 28 MB/s | 1101 MB/s | 70955653 | 33.48 | -| lizard 1.0 -46 | 6.25 MB/s | 1073 MB/s | 65413061 | 30.86 | -| lizard 1.0 -49 | 1.27 MB/s | 1064 MB/s | 60679215 | 28.63 | -| zlib 1.2.8 -1 | 66 MB/s | 244 MB/s | 77259029 | 36.45 | -| zlib 1.2.8 -6 | 20 MB/s | 263 MB/s | 68228431 | 32.19 | -| zlib 1.2.8 -9 | 8.37 MB/s | 266 MB/s | 67644548 | 31.92 | -| zstd 1.1.1 -1 | 235 MB/s | 645 MB/s | 73659468 | 34.75 | -| zstd 1.1.1 -2 | 181 MB/s | 600 MB/s | 70168955 | 33.11 | -| zstd 1.1.1 -5 | 88 MB/s | 565 MB/s | 65002208 | 30.67 | -| zstd 1.1.1 -8 | 31 MB/s | 619 MB/s | 61026497 | 28.79 | -| zstd 1.1.1 -11 | 16 MB/s | 613 MB/s | 59523167 | 28.08 | -| zstd 1.1.1 -15 | 4.97 MB/s | 639 MB/s | 58007773 | 27.37 | -| zstd 1.1.1 -18 | 2.87 MB/s | 583 MB/s | 55294241 | 26.09 | -| zstd 1.1.1 -22 | 1.44 MB/s | 505 MB/s | 52731930 | 24.88 | -| brotli 0.5.2 -0 | 217 MB/s | 244 MB/s | 78226979 | 36.91 | -| brotli 0.5.2 -2 | 96 MB/s | 283 MB/s | 68066621 | 32.11 | -| brotli 0.5.2 -5 | 24 MB/s | 312 MB/s | 60801716 | 28.69 | -| brotli 0.5.2 -8 | 5.56 MB/s | 324 MB/s | 57382470 | 27.07 | -| brotli 0.5.2 -11 | 0.39 MB/s | 266 MB/s | 51138054 | 24.13 | - -[silesia.tar]: https://drive.google.com/file/d/0BwX7dtyRLxThenZpYU9zLTZhR1k/view?usp=sharing - - -Documentation -------------------------- - -The raw Lizard block compression format is detailed within [lizard_Block_format]. - -To compress an arbitrarily long file or data stream, multiple blocks are required. -Organizing these blocks and providing a common header format to handle their content -is the purpose of the Frame format, defined into [lizard_Frame_format]. -Interoperable versions of Lizard must respect this frame format. - -[lizard_Block_format]: doc/lizard_Block_format.md -[lizard_Frame_format]: doc/lizard_Frame_format.md diff --git a/contrib/lizard/appveyor.yml b/contrib/lizard/appveyor.yml deleted file mode 100644 index d5863f7c06b..00000000000 --- a/contrib/lizard/appveyor.yml +++ /dev/null @@ -1,143 +0,0 @@ -version: 1.0.{build} -environment: - matrix: - - COMPILER: "visual" - CONFIGURATION: "Debug" - PLATFORM: "x64" - - COMPILER: "visual" - CONFIGURATION: "Debug" - PLATFORM: "Win32" - - COMPILER: "visual" - CONFIGURATION: "Release" - PLATFORM: "x64" - - COMPILER: "visual" - CONFIGURATION: "Release" - PLATFORM: "Win32" - - COMPILER: "gcc" - PLATFORM: "mingw64" - - COMPILER: "gcc" - PLATFORM: "mingw32" - - COMPILER: "gcc" - PLATFORM: "clang" - -install: - - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - - MKDIR bin - - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH% - - if [%COMPILER%]==[gcc] ( - SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && - SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && - COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe && - COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe - ) else ( - IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") - ) - -build_script: - - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - - if [%PLATFORM%]==[clang] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - - ECHO *** && - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% && - ECHO *** - - if [%PLATFORM%]==[clang] (clang -v) - - if [%COMPILER%]==[gcc] (gcc -v) - - if [%COMPILER%]==[gcc] ( - echo ----- && - make -v && - echo ----- && - if not [%PLATFORM%]==[clang] ( - make -C programs lizard && make -C tests fullbench && make -C lib lib - ) ELSE ( - make -C programs lizard CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" && - make -C tests fullbench CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" && - make -C lib lib CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" - ) - ) - - if [%COMPILER%]==[gccX] if not [%PLATFORM%]==[clang] ( - MKDIR bin\dll bin\static bin\example bin\include && - COPY tests\fullbench.c bin\example\ && - COPY lib\xxhash.c bin\example\ && - COPY lib\xxhash.h bin\example\ && - COPY lib\lizard.h bin\include\ && - COPY lib\lizardhc.h bin\include\ && - COPY lib\lizard_frame.h bin\include\ && - COPY lib\liblizard.a bin\static\liblizard_static.lib && - COPY lib\dll\liblizard.* bin\dll\ && - COPY lib\dll\example\Makefile bin\example\ && - COPY lib\dll\example\fullbench-dll.* bin\example\ && - COPY lib\dll\example\README.md bin\ && - COPY programs\lizard.exe bin\lizard.exe - ) - - if [%COMPILER%]==[gccX] if [%PLATFORM%]==[mingw64] ( - 7z.exe a bin\lizard_x64.zip NEWS .\bin\lizard.exe .\bin\README.md .\bin\example .\bin\dll .\bin\static .\bin\include && - appveyor PushArtifact bin\lizard_x64.zip - ) - - if [%COMPILER%]==[gccX] if [%PLATFORM%]==[mingw32] ( - 7z.exe a bin\lizard_x86.zip NEWS .\bin\lizard.exe .\bin\README.md .\bin\example .\bin\dll .\bin\static .\bin\include && - appveyor PushArtifact bin\lizard_x86.zip - ) - - if [%COMPILER%]==[gcc] (COPY tests\fullbench.exe programs\) - - if [%COMPILER%]==[visual] ( - ECHO *** && - ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && - ECHO *** && - msbuild "visual\VS2010\lizard.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - ECHO *** && - ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && - ECHO *** && - msbuild "visual\VS2010\lizard.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - ECHO *** && - ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% && - ECHO *** && - msbuild "visual\VS2010\lizard.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - ECHO *** && - ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && - ECHO *** && - msbuild "visual\VS2010\lizard.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && - COPY visual\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe programs\ - ) - -test_script: - - ECHO *** && - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% && - ECHO *** - - if not [%COMPILER%]==[unknown] ( - CD programs && - lizard -h && - lizard -i1b lizard.exe && - lizard -i1b5 lizard.exe && - lizard -i1b10 lizard.exe && - lizard -i1b15 lizard.exe && - echo ------- lizard tested ------- && - fullbench.exe -i1 fullbench.exe - ) - -artifacts: - - path: bin\lizard_x64.zip - - path: bin\lizard_x86.zip - -deploy: -- provider: GitHub - artifact: bin\lizard_x64.zip - auth_token: - secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4 - force_update: true - prerelease: true - on: - COMPILER: gcc - PLATFORM: "mingw64" - appveyor_repo_tag: true - branch: autobuild - -- provider: GitHub - artifact: bin\lizard_x86.zip - auth_token: - secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4 - force_update: true - prerelease: true - on: - COMPILER: gcc - PLATFORM: "mingw32" - appveyor_repo_tag: true - branch: autobuild diff --git a/contrib/lizard/contrib/djgpp/LICENSE b/contrib/lizard/contrib/djgpp/LICENSE deleted file mode 100644 index fee0d3b3e23..00000000000 --- a/contrib/lizard/contrib/djgpp/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ -Copyright (c) 2014, lpsantil -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/contrib/lizard/contrib/djgpp/README.MD b/contrib/lizard/contrib/djgpp/README.MD deleted file mode 100644 index 8dfc3a838c4..00000000000 --- a/contrib/lizard/contrib/djgpp/README.MD +++ /dev/null @@ -1,21 +0,0 @@ -# lizard for DOS/djgpp -This file details on how to compile lizard.exe, and liblizard.a for use on DOS/djgpp using -Andrew Wu's build-djgpp cross compilers ([GH][0], [Binaries][1]) on OSX, Linux. - -## Setup -* Download a djgpp tarball [binaries][1] for your platform. -* Extract and install it (`tar jxvf djgpp-linux64-gcc492.tar.bz2`). Note the path. We'll assume `/home/user/djgpp`. -* Add the `bin` folder to your `PATH`. In bash, do `export PATH=/home/user/djgpp/bin:$PATH`. -* The `Makefile` in `contrib/djgpp/` sets up `CC`, `AR`, `LD` for you. So, `CC=i586-pc-msdosdjgpp-gcc`, `AR=i586-pc-msdosdjgpp-ar`, `LD=i586-pc-msdosdjgpp-gcc`. - -## Building Lizard for DOS -In the base dir of lizard and with `contrib/djgpp/Makefile`, try: -Try: -* `make -f contrib/djgpp/Makefile` -* `make -f contrib/djgpp/Makefile liblizard.a` -* `make -f contrib/djgpp/Makefile lizard.exe` -* `make -f contrib/djgpp/Makefile DESTDIR=/home/user/dos install`, however it doesn't make much sense on a \*nix. -* You can also do `make -f contrib/djgpp/Makefile uninstall` - -[0]: https://github.com/andrewwutw/build-djgpp -[1]: https://github.com/andrewwutw/build-djgpp/releases diff --git a/contrib/lizard/doc/lizard_Block_format.md b/contrib/lizard/doc/lizard_Block_format.md deleted file mode 100644 index 45307a1fce5..00000000000 --- a/contrib/lizard/doc/lizard_Block_format.md +++ /dev/null @@ -1,181 +0,0 @@ -Lizard v1.x Block Format Description -============================ -Last revised: 2016-10-08 -Authors : Yann Collet, Przemyslaw Skibinski - - -This specification is intended for developers -willing to produce Lizard-compatible compressed data blocks -using any programming language. - -Lizard is an LZ77-type compressor with a fixed, byte-oriented encoding. -There is no framing layer as it is assumed to be handled by other parts of the system (see [Lizard Frame format]). -This design is assumed to favor simplicity and speed. -It helps later on for optimizations, compactness, and features. - -This document describes only the block format, -not how the compressor nor decompressor actually work. -The correctness of the decompressor should not depend -on implementation details of the compressor, and vice versa. - -[Lizard Frame format]: lizard_Frame_format.md - - -Division into blocks --------------------- - -The input data is divided into blocks of maximum size LIZARD_BLOCK_SIZE (which is 128 KB). The subsequent blocks use the same sliding window and are dependent on previous blocks. -Our impementation of Lizard compressor divides input data into blocks of size of LIZARD_BLOCK_SIZE except the last one which usually will be smaller. -The output data is a single byte 'Compression_Level' and one or more blocks in the format described below. - - -Block header format ------------------------ - -The block header is a single byte `Header_Byte` that is combination of following flags: - -| Name | Value | -| --------------------- | --- | -| LIZARD_FLAG_LITERALS | 1 | -| LIZARD_FLAG_FLAGS | 2 | -| LIZARD_FLAG_OFF16LEN | 4 | -| LIZARD_FLAG_OFF24LEN | 8 | -| LIZARD_FLAG_LEN | 16 | -| LIZARD_FLAG_UNCOMPRESSED | 128 | - -When `Header_Byte & LIZARD_FLAG_UNCOMPRESSED` is true then the block is followed by 3-byte `Uncompressed_length` and uncompressed data of given size. - - -Compressed block content ------------------------- - -When `Header_Byte & LIZARD_FLAG_UNCOMPRESSED` is false then compressed block contains of 5 streams: -- `Lengths_Stream` (compressed with Huffman if LIZARD_FLAG_LEN is set) -- `16-bit_Offsets_Stream` (compressed with Huffman if LIZARD_FLAG_OFF16LEN is set) -- `24-bit_Offsets_Stream` (compressed with Huffman if LIZARD_FLAG_OFF24LEN is set) -- `Tokens_Stream` (compressed with Huffman if LIZARD_FLAG_FLAGS is set) -- `Literals_Stream` (compressed with Huffman if LIZARD_FLAG_LITERALS is set) - - -Stream format -------------- -The single stream is either: -- if LIZARD_FLAG_XXX is not set: 3 byte `Stream_Length` followed by a given number bytes -- if LIZARD_FLAG_XXX is set: 3 byte `Original_Stream_Length`, 3 byte `Compressed_Stream_Length`, followed by a given number of Huffman compressed bytes - - -Lizard block decompression ------------------------ -At the beginning we have 5 streams and their sizes. -Decompressor should iterate through `Tokens_Stream`. Each token is 1-byte long and describes how to get data from other streams. -If token points a stream that is already empty it means that data is corrupted. - - -Lizard token decompression ------------------------ -The token is a one byte. Token decribes: -- how many literals should be copied from `Literals_Stream` -- if offset should be read from `16-bit_Offsets_Stream` or `24-bit_Offsets_Stream` -- how many bytes are part of a match and should be copied from a sliding window - -Lizard uses 4 types of tokens: -- [0_MMMM_LLL] - 3-bit literal length (0-7+), use offset from `16-bit_Offsets_Stream`, 4-bit match length (4-15+) -- [1_MMMM_LLL] - 3-bit literal length (0-7+), use last offset, 4-bit match length (0-15+) -- token 31 - no literal length, use offset from `24-bit_Offsets_Stream`, match length (47+) -- token 0-30 - no literal length, use offset from `24-bit_Offsets_Stream`, 31 match lengths (16-46) - -Lizard uses different output codewords and is not compatible with LZ4. LZ4 output codewords are 3 byte long (24-bit) and look as follows: -- LLLL_MMMM OOOOOOOO OOOOOOOO - 16-bit offset, 4-bit match length, 4-bit literal length - - -The format of `Lengths_Stream` ------------------------------- -`Lengths_Stream` contains lenghts in the the following format: -- when 'First_Byte' is < 254 then lenght is equal 'First_Byte' -- when 'First_Byte' is 254 then lenght is equal to value of 2-bytes after 'First_Byte' i.e. 0-65536 -- when 'First_Byte' is 255 then lenght is equal to value of 3-bytes after 'First_Byte' i.e. 0-16777215 - - -[0_MMMM_LLL] and [1_MMMM_LLL] tokens ---------------------------------------- -The length of literals to be copied from `Literals_Stream` depends on the literal length field (LLL) that uses 3 bits of the token. -Therefore each field ranges from 0 to 7. -If the value is 7, then the lenght is increased with a length taken from `Lengths_Stream`. - -Example 1 : A literal length of 48 will be represented as : - - - 7 : value for the 3-bits LLL field - - 41 : (=48-7) remaining length to reach 48 (in `Lengths_Stream`) - -Example 2 : A literal length of 280 for will be represented as : - - - 7 : value for the 3-bits LLL field - - 254 : informs that remaining length (=280-7) must be represented as 2-bytes (in `Lengths_Stream`) - - 273 : (=280-7) encoded as 2-bytes (in `Lengths_Stream`) - -Example 3 : A literal length of 7 for will be represented as : - - - 7 : value for the 3-bits LLL field - - 0 : (=7-7) yes, the zero must be output (in `Lengths_Stream`) - -After copying 0 or more literals from `Literals_Stream` we can prepare the match copy operation which depends on a offset and a match length. -The flag "0" informs that decoder should use the last encoded offset. -The flag "1" informs that the offset is a 2 bytes value (16-bit), in little endian format and should be taken from `16-bit_Offsets_Stream`. - -The match length depends on the match length field (MMMM) that uses 4 bits of the token. -Therefore each field ranges from 0 to 15. Values from 0-3 are forbidden with offset taken from `16-bit_Offsets_Stream`. -If the value is 15, then the lenght is increased with a length taken from `Lengths_Stream`. - -With the offset and the match length, -the decoder can now proceed to copy the data from the already decoded buffer. - - -Lizard block epilogue ------------------- -When all tokens are read from `Tokens_Stream` and interpreted all remaining streams should also be empty. -Otherwise, it means that the data is corrupted. The only exception is `Literals_Stream` that should have at least 16 remaining literals what -allows fast memory copy operations. The remaining literals up to the end of `Literals_Stream` should be appended to the output data. - - -Tokens 0-31 ------------ -The offset is a 3 bytes value (24-bit), in little endian format and should be taken from `24-bit_Offsets_Stream`. -The offset represents the position of the match to be copied from. -1 means "current position - 1 byte". -The maximum offset value is (1<<24)-1, 1<<24 cannot be coded. -Note that 0 is an invalid value, not used. - -The 'Token_Value' ranges from 0 to 31. -The match length is equal to 'Token_Value + 16 that is from 16 to 47. -If match length is 47, the lenght is increased with a length taken from `Lengths_Stream`. - - -Parsing restrictions ------------------------ -There are specific parsing rules to respect in order to remain compatible -with assumptions made by the decoder : - -1. The last 16 bytes are always literals what allows fast memory copy operations. -2. The last match must start at least 20 bytes before end of block. - Consequently, a block with less than 20 bytes cannot be compressed. - -These rules are in place to ensure that the decoder -will never read beyond the input buffer, nor write beyond the output buffer. - -Note that the last sequence is also incomplete, -and stops right after literals. - - -Additional notes ------------------------ -There is no assumption nor limits to the way the compressor -searches and selects matches within the source data block. -It could be a fast scan, a multi-probe, a full search using BST, -standard hash chains or MMC, well whatever. - -Advanced parsing strategies can also be implemented, such as lazy match, -or full optimal parsing. - -All these trade-off offer distinctive speed/memory/compression advantages. -Whatever the method used by the compressor, its result will be decodable -by any Lizard decoder if it follows the format specification described above. diff --git a/contrib/lizard/doc/lizard_Frame_format.md b/contrib/lizard/doc/lizard_Frame_format.md deleted file mode 100644 index ce24b0e23e2..00000000000 --- a/contrib/lizard/doc/lizard_Frame_format.md +++ /dev/null @@ -1,312 +0,0 @@ -Lizard v1.x Frame Format Description -================================= - -###Notices - -Copyright (c) 2013-2015 Yann Collet - -Copyright (c) 2016 Przemyslaw Skibinski - -Permission is granted to copy and distribute this document -for any purpose and without charge, -including translations into other languages -and incorporation into compilations, -provided that the copyright notice and this notice are preserved, -and that any substantive changes or deletions from the original -are clearly marked. -Distribution of this document is unlimited. - -###Version - -1.0 (8-10-2016) - - -Introduction ------------- - -The purpose of this document is to define a lossless compressed data format, -that is independent of CPU type, operating system, -file system and character set, suitable for -File compression, Pipe and streaming compression -using the Lizard algorithm. - -The data can be produced or consumed, -even for an arbitrarily long sequentially presented input data stream, -using only an a priori bounded amount of intermediate storage, -and hence can be used in data communications. -The format uses the Lizard compression method, -and optional [xxHash-32 checksum method](https://github.com/Cyan4973/xxHash), -for detection of data corruption. - -The data format defined by this specification -does not attempt to allow random access to compressed data. - -This specification is intended for use by implementers of software -to compress data into Lizard format and/or decompress data from Lizard format. -The text of the specification assumes a basic background in programming -at the level of bits and other primitive data representations. - -Unless otherwise indicated below, -a compliant compressor must produce data sets -that conform to the specifications presented here. -It doesn’t need to support all options though. - -A compliant decompressor must be able to decompress -at least one working set of parameters -that conforms to the specifications presented here. -It may also ignore checksums. -Whenever it does not support a specific parameter within the compressed stream, -it must produce a non-ambiguous error code -and associated error message explaining which parameter is unsupported. - - -General Structure of Lizard Frame format -------------------------------------- - -| MagicNb | F. Descriptor | Block | (...) | EndMark | C. Checksum | -|:-------:|:-------------:| ----- | ----- | ------- | ----------- | -| 4 bytes | 3-11 bytes | | | 4 bytes | 0-4 bytes | - -__Magic Number__ - -4 Bytes, Little endian format. -Value : 0x184D2206 (it was 0x184D2204 for LZ4 and 0x184D2205 for LZ5 v1.x) - -__Frame Descriptor__ - -3 to 11 Bytes, to be detailed in the next part. -Most important part of the spec. - -__Data Blocks__ - -To be detailed later on. -That’s where compressed data is stored. - -__EndMark__ - -The flow of blocks ends when the last data block has a size of “0”. -The size is expressed as a 32-bits value. - -__Content Checksum__ - -Content Checksum verify that the full content has been decoded correctly. -The content checksum is the result -of [xxh32() hash function](https://github.com/Cyan4973/xxHash) -digesting the original (decoded) data as input, and a seed of zero. -Content checksum is only present when its associated flag -is set in the frame descriptor. -Content Checksum validates the result, -that all blocks were fully transmitted in the correct order and without error, -and also that the encoding/decoding process itself generated no distortion. -Its usage is recommended. - -__Frame Concatenation__ - -In some circumstances, it may be preferable to append multiple frames, -for example in order to add new data to an existing compressed file -without re-framing it. - -In such case, each frame has its own set of descriptor flags. -Each frame is considered independent. -The only relation between frames is their sequential order. - -The ability to decode multiple concatenated frames -within a single stream or file -is left outside of this specification. -As an example, the reference lizard command line utility behavior is -to decode all concatenated frames in their sequential order. - - -Frame Descriptor ----------------- - -| FLG | BD | (Content Size) | HC | -| ------- | ------- |:--------------:| ------- | -| 1 byte | 1 byte | 0 - 8 bytes | 1 byte | - -The descriptor uses a minimum of 3 bytes, -and up to 11 bytes depending on optional parameters. - -__FLG byte__ - -| BitNb | 7-6 | 5 | 4 | 3 | 2 | 1-0 | -| ------- | ------- | ------- | --------- | ------- | --------- | -------- | -|FieldName| Version | B.Indep | B.Checksum| C.Size | C.Checksum|*Reserved*| - - -__BD byte__ - -| BitNb | 7 | 6-5-4 | 3-2-1-0 | -| ------- | -------- | ------------ | -------- | -|FieldName|*Reserved*| Block MaxSize|*Reserved*| - -In the tables, bit 7 is highest bit, while bit 0 is lowest. - -__Version Number__ - -2-bits field, must be set to “01”. -Any other value cannot be decoded by this version of the specification. -Other version numbers will use different flag layouts. - -__Block Independence flag__ - -If this flag is set to “1”, blocks are independent. -If this flag is set to “0”, each block depends on previous ones -(up to Lizard window size, which is 16 MB). -In such case, it’s necessary to decode all blocks in sequence. - -Block dependency improves compression ratio, especially for small blocks. -On the other hand, it makes direct jumps or multi-threaded decoding impossible. - -__Block checksum flag__ - -If this flag is set, each data block will be followed by a 4-bytes checksum, -calculated by using the xxHash-32 algorithm on the raw (compressed) data block. -The intention is to detect data corruption (storage or transmission errors) -immediately, before decoding. -Block checksum usage is optional. - -__Content Size flag__ - -If this flag is set, the uncompressed size of data included within the frame -will be present as an 8 bytes unsigned little endian value, after the flags. -Content Size usage is optional. - -__Content checksum flag__ - -If this flag is set, a content checksum will be appended after the EndMark. - -Recommended value : “1” (content checksum is present) - -__Block Maximum Size__ - -This information is intended to help the decoder allocate memory. -Size here refers to the original (uncompressed) data size. -Block Maximum Size is one value among the following table : - -| 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | -| --- | ------ | ------ | ---- | ---- | ----- | ----- | ------ | -| N/A | 128 KB | 256 KB | 1 MB | 4 MB | 16 MB | 64 MB | 256 MB | - - -The decoder may refuse to allocate block sizes above a (system-specific) size. -Unused values may be used in a future revision of the spec. -A decoder conformant to the current version of the spec -is only able to decode blocksizes defined in this spec. - -__Reserved bits__ - -Value of reserved bits **must** be 0 (zero). -Reserved bit might be used in a future version of the specification, -typically enabling new optional features. -If this happens, a decoder respecting the current version of the specification -shall not be able to decode such a frame. - -__Content Size__ - -This is the original (uncompressed) size. -This information is optional, and only present if the associated flag is set. -Content size is provided using unsigned 8 Bytes, for a maximum of 16 HexaBytes. -Format is Little endian. -This value is informational, typically for display or memory allocation. -It can be skipped by a decoder, or used to validate content correctness. - -__Header Checksum__ - -One-byte checksum of combined descriptor fields, including optional ones. -The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF ` -using zero as a seed, -and the full Frame Descriptor as an input -(including optional fields when they are present). -A wrong checksum indicates an error in the descriptor. -Header checksum is informational and can be skipped. - - -Data Blocks ------------ - -| Block Size | data | (Block Checksum) | -|:----------:| ------ |:----------------:| -| 4 bytes | | 0 - 4 bytes | - - -__Block Size__ - -This field uses 4-bytes, format is little-endian. - -The highest bit is “1” if data in the block is uncompressed. - -The highest bit is “0” if data in the block is compressed by Lizard. - -All other bits give the size, in bytes, of the following data block -(the size does not include the block checksum if present). - -Block Size shall never be larger than Block Maximum Size. -Such a thing could happen for incompressible source data. -In such case, such a data block shall be passed in uncompressed format. - -__Data__ - -Where the actual data to decode stands. -It might be compressed or not, depending on previous field indications. -Uncompressed size of Data can be any size, up to “block maximum size”. -Note that data block is not necessarily full : -an arbitrary “flush” may happen anytime. Any block can be “partially filled”. - -__Block checksum__ - -Only present if the associated flag is set. -This is a 4-bytes checksum value, in little endian format, -calculated by using the xxHash-32 algorithm on the raw (undecoded) data block, -and a seed of zero. -The intention is to detect data corruption (storage or transmission errors) -before decoding. - -Block checksum is cumulative with Content checksum. - - -Skippable Frames ----------------- - -| Magic Number | Frame Size | User Data | -|:------------:|:----------:| --------- | -| 4 bytes | 4 bytes | | - -Skippable frames allow the integration of user-defined data -into a flow of concatenated frames. -Its design is pretty straightforward, -with the sole objective to allow the decoder to quickly skip -over user-defined data and continue decoding. - -For the purpose of facilitating identification, -it is discouraged to start a flow of concatenated frames with a skippable frame. -If there is a need to start such a flow with some user data -encapsulated into a skippable frame, -it’s recommended to start with a zero-byte Lizard frame -followed by a skippable frame. -This will make it easier for file type identifiers. - - -__Magic Number__ - -4 Bytes, Little endian format. -Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F. -All 16 values are valid to identify a skippable frame. - -__Frame Size__ - -This is the size, in bytes, of the following User Data -(without including the magic number nor the size field itself). -4 Bytes, Little endian format, unsigned 32-bits. -This means User Data can’t be bigger than (2^32-1) Bytes. - -__User Data__ - -User Data can be anything. Data will just be skipped by the decoder. - - - -Version changes ---------------- - -1.0 : based on LZ4 Frame Format Description 1.5.1 (31/03/2015) diff --git a/contrib/lizard/examples/.gitignore b/contrib/lizard/examples/.gitignore deleted file mode 100644 index f6f8bbcca09..00000000000 --- a/contrib/lizard/examples/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -/Makefile.liz* -/printVersion -/doubleBuffer -/ringBuffer -/ringBufferHC -/lineCompress -/frameCompress -/*.exe diff --git a/contrib/lizard/examples/HCStreaming_ringBuffer.c b/contrib/lizard/examples/HCStreaming_ringBuffer.c deleted file mode 100644 index 3e4d75a5d78..00000000000 --- a/contrib/lizard/examples/HCStreaming_ringBuffer.c +++ /dev/null @@ -1,241 +0,0 @@ -// Lizard streaming API example : ring buffer -// Based on previous work from Takayuki Matsuoka - - -/************************************** - * Compiler Options - **************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS // for MSVC -# define snprintf sprintf_s -#endif - -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif - - -/************************************** - * Includes - **************************************/ -#include "lizard_compress.h" -#include "lizard_decompress.h" -#include "lizard_common.h" - -#include -#include -#include -#include - -enum { - MESSAGE_MAX_BYTES = 1024, - RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES, - DEC_BUFFER_BYTES = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES // Intentionally larger to test unsynchronized ring buffers -}; - - -size_t write_int32(FILE* fp, int32_t i) { - return fwrite(&i, sizeof(i), 1, fp); -} - -size_t write_bin(FILE* fp, const void* array, int arrayBytes) { - return fwrite(array, 1, arrayBytes, fp); -} - -size_t read_int32(FILE* fp, int32_t* i) { - return fread(i, sizeof(*i), 1, fp); -} - -size_t read_bin(FILE* fp, void* array, int arrayBytes) { - return fread(array, 1, arrayBytes, fp); -} - - -void test_compress(FILE* outFp, FILE* inpFp) -{ - Lizard_stream_t* lizardStream = Lizard_createStream(0); - static char inpBuf[RING_BUFFER_BYTES]; - int inpOffset = 0; - - if (!lizardStream) return; - - for(;;) - { - // Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer. - char* const inpPtr = &inpBuf[inpOffset]; - const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1; - const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); - if (0 == inpBytes) break; - - { - char cmpBuf[LIZARD_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; - const int cmpBytes = Lizard_compress_continue(lizardStream, inpPtr, cmpBuf, inpBytes, Lizard_compressBound(inpBytes)); - - if(cmpBytes <= 0) break; - write_int32(outFp, cmpBytes); - write_bin(outFp, cmpBuf, cmpBytes); - - inpOffset += inpBytes; - - // Wraparound the ringbuffer offset - if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) - inpOffset = 0; - } - } - - write_int32(outFp, 0); - Lizard_freeStream(lizardStream); -} - - -void test_decompress(FILE* outFp, FILE* inpFp) -{ - static char decBuf[DEC_BUFFER_BYTES]; - int decOffset = 0; - Lizard_streamDecode_t lizardStreamDecode_body = { 0 }; - Lizard_streamDecode_t* lizardStreamDecode = &lizardStreamDecode_body; - - for(;;) - { - int cmpBytes = 0; - char cmpBuf[LIZARD_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; - - { - const size_t r0 = read_int32(inpFp, &cmpBytes); - size_t r1; - if(r0 != 1 || cmpBytes <= 0) - break; - - r1 = read_bin(inpFp, cmpBuf, cmpBytes); - if(r1 != (size_t) cmpBytes) - break; - } - - { - char* const decPtr = &decBuf[decOffset]; - const int decBytes = Lizard_decompress_safe_continue( - lizardStreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES); - if(decBytes <= 0) - break; - - decOffset += decBytes; - write_bin(outFp, decPtr, decBytes); - - // Wraparound the ringbuffer offset - if(decOffset >= DEC_BUFFER_BYTES - MESSAGE_MAX_BYTES) - decOffset = 0; - } - } -} - - -// Compare 2 files content -// return 0 if identical -// return ByteNb>0 if different -size_t compare(FILE* f0, FILE* f1) -{ - size_t result = 1; - - for (;;) - { - char b0[65536]; - char b1[65536]; - const size_t r0 = fread(b0, 1, sizeof(b0), f0); - const size_t r1 = fread(b1, 1, sizeof(b1), f1); - - if ((r0==0) && (r1==0)) return 0; // success - - if (r0 != r1) - { - size_t smallest = r0; - if (r1 -#include -#include -#include - -enum { - BLOCK_BYTES = 1024 * 8, -// BLOCK_BYTES = 1024 * 64, -}; - - -size_t write_int(FILE* fp, int i) { - return fwrite(&i, sizeof(i), 1, fp); -} - -size_t write_bin(FILE* fp, const void* array, size_t arrayBytes) { - return fwrite(array, 1, arrayBytes, fp); -} - -size_t read_int(FILE* fp, int* i) { - return fread(i, sizeof(*i), 1, fp); -} - -size_t read_bin(FILE* fp, void* array, size_t arrayBytes) { - return fread(array, 1, arrayBytes, fp); -} - - -void test_compress(FILE* outFp, FILE* inpFp) -{ - Lizard_stream_t* lizardStream = Lizard_createStream_MinLevel(); - char inpBuf[2][BLOCK_BYTES]; - int inpBufIndex = 0; - - if (!lizardStream) return; - lizardStream = Lizard_resetStream_MinLevel(lizardStream); - if (!lizardStream) return; - - for(;;) { - char* const inpPtr = inpBuf[inpBufIndex]; - const int inpBytes = (int) read_bin(inpFp, inpPtr, BLOCK_BYTES); - if(0 == inpBytes) { - break; - } - - { - char cmpBuf[LIZARD_COMPRESSBOUND(BLOCK_BYTES)]; - const int cmpBytes = Lizard_compress_continue(lizardStream, inpPtr, cmpBuf, inpBytes, sizeof(cmpBuf)); - if(cmpBytes <= 0) { - break; - } - write_int(outFp, cmpBytes); - write_bin(outFp, cmpBuf, (size_t) cmpBytes); - } - - inpBufIndex = (inpBufIndex + 1) % 2; - } - - write_int(outFp, 0); - Lizard_freeStream(lizardStream); -} - - -void test_decompress(FILE* outFp, FILE* inpFp) -{ - Lizard_streamDecode_t lizardStreamDecode_body; - Lizard_streamDecode_t* lizardStreamDecode = &lizardStreamDecode_body; - - char decBuf[2][BLOCK_BYTES]; - int decBufIndex = 0; - - Lizard_setStreamDecode(lizardStreamDecode, NULL, 0); - - for(;;) { - char cmpBuf[LIZARD_COMPRESSBOUND(BLOCK_BYTES)]; - int cmpBytes = 0; - - { - const size_t readCount0 = read_int(inpFp, &cmpBytes); - if(readCount0 != 1 || cmpBytes <= 0) { - break; - } - - const size_t readCount1 = read_bin(inpFp, cmpBuf, (size_t) cmpBytes); - if(readCount1 != (size_t) cmpBytes) { - break; - } - } - - { - char* const decPtr = decBuf[decBufIndex]; - const int decBytes = Lizard_decompress_safe_continue( - lizardStreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES); - if(decBytes <= 0) { - break; - } - write_bin(outFp, decPtr, (size_t) decBytes); - } - - decBufIndex = (decBufIndex + 1) % 2; - } -} - - -int compare(FILE* fp0, FILE* fp1) -{ - int result = 0; - - while(0 == result) { - char b0[65536]; - char b1[65536]; - const size_t r0 = read_bin(fp0, b0, sizeof(b0)); - const size_t r1 = read_bin(fp1, b1, sizeof(b1)); - - result = (int) r0 - (int) r1; - - if(0 == r0 || 0 == r1) { - break; - } - if(0 == result) { - result = memcmp(b0, b1, r0); - } - } - - return result; -} - - -int main(int argc, char* argv[]) -{ - char inpFilename[256] = { 0 }; - char lizardFilename[256] = { 0 }; - char decFilename[256] = { 0 }; - - if(argc < 2) { - printf("Please specify input filename\n"); - return 0; - } - - snprintf(inpFilename, 256, "%s", argv[1]); - snprintf(lizardFilename, 256, "%s.lizs-%d", argv[1], BLOCK_BYTES); - snprintf(decFilename, 256, "%s.lizs-%d.dec", argv[1], BLOCK_BYTES); - - printf("inp = [%s]\n", inpFilename); - printf("lizard = [%s]\n", lizardFilename); - printf("dec = [%s]\n", decFilename); - - // compress - { - FILE* inpFp = fopen(inpFilename, "rb"); - FILE* outFp = fopen(lizardFilename, "wb"); - - printf("compress : %s -> %s\n", inpFilename, lizardFilename); - test_compress(outFp, inpFp); - printf("compress : done\n"); - - fclose(outFp); - fclose(inpFp); - } - - // decompress - { - FILE* inpFp = fopen(lizardFilename, "rb"); - FILE* outFp = fopen(decFilename, "wb"); - - printf("decompress : %s -> %s\n", lizardFilename, decFilename); - test_decompress(outFp, inpFp); - printf("decompress : done\n"); - - fclose(outFp); - fclose(inpFp); - } - - // verify - { - FILE* inpFp = fopen(inpFilename, "rb"); - FILE* decFp = fopen(decFilename, "rb"); - - printf("verify : %s <-> %s\n", inpFilename, decFilename); - const int cmp = compare(inpFp, decFp); - if(0 == cmp) { - printf("verify : OK\n"); - } else { - printf("verify : NG\n"); - } - - fclose(decFp); - fclose(inpFp); - } - - return 0; -} diff --git a/contrib/lizard/examples/blockStreaming_doubleBuffer.md b/contrib/lizard/examples/blockStreaming_doubleBuffer.md deleted file mode 100644 index c27afcfb98c..00000000000 --- a/contrib/lizard/examples/blockStreaming_doubleBuffer.md +++ /dev/null @@ -1,100 +0,0 @@ -# Lizard streaming API Example : Double Buffer -by *Takayuki Matsuoka* - -`blockStreaming_doubleBuffer.c` is Lizard Straming API example which implements double buffer (de)compression. - -Please note : - - - Firstly, read "Lizard Streaming API Basics". - - This is relatively advanced application example. - - Output file is not compatible with lizard_frame and platform dependent. - - -## What's the point of this example ? - - - Handle huge file in small amount of memory - - Always better compression ratio than Block API - - Uniform block size - - -## How the compression works - -First of all, allocate "Double Buffer" for input and Lizard compressed data buffer for output. -Double buffer has two pages, "first" page (Page#1) and "second" page (Page#2). - -``` - Double Buffer - - Page#1 Page#2 - +---------+---------+ - | Block#1 | | - +----+----+---------+ - | - v - {Out#1} - - - Prefix Dependency - +---------+ - | | - v | - +---------+----+----+ - | Block#1 | Block#2 | - +---------+----+----+ - | - v - {Out#2} - - - External Dictionary Mode - +---------+ - | | - | v - +----+----+---------+ - | Block#3 | Block#2 | - +----+----+---------+ - | - v - {Out#3} - - - Prefix Dependency - +---------+ - | | - v | - +---------+----+----+ - | Block#3 | Block#4 | - +---------+----+----+ - | - v - {Out#4} -``` - -Next, read first block to double buffer's first page. And compress it by `Lizard_compress_continue()`. -For the first time, Lizard doesn't know any previous dependencies, -so it just compress the line without dependencies and generates compressed block {Out#1} to Lizard compressed data buffer. -After that, write {Out#1} to the file. - -Next, read second block to double buffer's second page. And compress it. -In this time, Lizard can use dependency to Block#1 to improve compression ratio. -This dependency is called "Prefix mode". - -Next, read third block to double buffer's *first* page. And compress it. -Also this time, Lizard can use dependency to Block#2. -This dependency is called "External Dictonaly mode". - -Continue these procedure to the end of the file. - - -## How the decompression works - -Decompression will do reverse order. - - - Read first compressed block. - - Decompress it to the first page and write that page to the file. - - Read second compressed block. - - Decompress it to the second page and write that page to the file. - - Read third compressed block. - - Decompress it to the *first* page and write that page to the file. - -Continue these procedure to the end of the compressed file. diff --git a/contrib/lizard/examples/blockStreaming_lineByLine.c b/contrib/lizard/examples/blockStreaming_lineByLine.c deleted file mode 100644 index 9d239533501..00000000000 --- a/contrib/lizard/examples/blockStreaming_lineByLine.c +++ /dev/null @@ -1,210 +0,0 @@ -// Lizard streaming API example : line-by-line logfile compression -// Copyright : Takayuki Matsuoka - - -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS -# define snprintf sprintf_s -#endif -#include "lizard_common.h" -#include "lizard_decompress.h" -#include -#include -#include -#include - -static size_t write_uint16(FILE* fp, uint16_t i) -{ - return fwrite(&i, sizeof(i), 1, fp); -} - -static size_t write_bin(FILE* fp, const void* array, int arrayBytes) -{ - return fwrite(array, 1, arrayBytes, fp); -} - -static size_t read_uint16(FILE* fp, uint16_t* i) -{ - return fread(i, sizeof(*i), 1, fp); -} - -static size_t read_bin(FILE* fp, void* array, int arrayBytes) -{ - return fread(array, 1, arrayBytes, fp); -} - - -static void test_compress( - FILE* outFp, - FILE* inpFp, - size_t messageMaxBytes, - size_t ringBufferBytes) -{ - Lizard_stream_t* const lizardStream = Lizard_createStream_MinLevel(); - const size_t cmpBufBytes = LIZARD_COMPRESSBOUND(messageMaxBytes); - char* const cmpBuf = (char*) malloc(cmpBufBytes); - char* const inpBuf = (char*) malloc(ringBufferBytes); - int inpOffset = 0; - - for ( ; ; ) - { - char* const inpPtr = &inpBuf[inpOffset]; - -#if 0 - // Read random length data to the ring buffer. - const int randomLength = (rand() % messageMaxBytes) + 1; - const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); - if (0 == inpBytes) break; -#else - // Read line to the ring buffer. - int inpBytes = 0; - if (!fgets(inpPtr, (int) messageMaxBytes, inpFp)) - break; - inpBytes = (int) strlen(inpPtr); -#endif - - { - const int cmpBytes = Lizard_compress_continue(lizardStream, inpPtr, cmpBuf, inpBytes, cmpBufBytes); - if (cmpBytes <= 0) break; - write_uint16(outFp, (uint16_t) cmpBytes); - write_bin(outFp, cmpBuf, cmpBytes); - - // Add and wraparound the ringbuffer offset - inpOffset += inpBytes; - if ((size_t)inpOffset >= ringBufferBytes - messageMaxBytes) inpOffset = 0; - } - } - write_uint16(outFp, 0); - - free(inpBuf); - free(cmpBuf); - Lizard_freeStream(lizardStream); -} - - -static void test_decompress( - FILE* outFp, - FILE* inpFp, - size_t messageMaxBytes, - size_t ringBufferBytes) -{ - Lizard_streamDecode_t* const lizardStreamDecode = Lizard_createStreamDecode(); - char* const cmpBuf = (char*) malloc(LIZARD_COMPRESSBOUND(messageMaxBytes)); - char* const decBuf = (char*) malloc(ringBufferBytes); - int decOffset = 0; - - for ( ; ; ) - { - uint16_t cmpBytes = 0; - - if (read_uint16(inpFp, &cmpBytes) != 1) break; - if (cmpBytes <= 0) break; - if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break; - - { - char* const decPtr = &decBuf[decOffset]; - const int decBytes = Lizard_decompress_safe_continue( - lizardStreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes); - if (decBytes <= 0) break; - write_bin(outFp, decPtr, decBytes); - - // Add and wraparound the ringbuffer offset - decOffset += decBytes; - if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0; - } - } - - free(decBuf); - free(cmpBuf); - Lizard_freeStreamDecode(lizardStreamDecode); -} - - -static int compare(FILE* f0, FILE* f1) -{ - int result = 0; - const size_t tempBufferBytes = 65536; - char* const b0 = (char*) malloc(tempBufferBytes); - char* const b1 = (char*) malloc(tempBufferBytes); - - while(0 == result) - { - const size_t r0 = fread(b0, 1, tempBufferBytes, f0); - const size_t r1 = fread(b1, 1, tempBufferBytes, f1); - - result = (int) r0 - (int) r1; - - if (0 == r0 || 0 == r1) break; - if (0 == result) result = memcmp(b0, b1, r0); - } - - free(b1); - free(b0); - return result; -} - - -int main(int argc, char* argv[]) -{ - enum { - MESSAGE_MAX_BYTES = 1024, - RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES, - }; - - char inpFilename[256] = { 0 }; - char lizardFilename[256] = { 0 }; - char decFilename[256] = { 0 }; - - if (argc < 2) - { - printf("Please specify input filename\n"); - return 0; - } - - snprintf(inpFilename, 256, "%s", argv[1]); - snprintf(lizardFilename, 256, "%s.lizs", argv[1]); - snprintf(decFilename, 256, "%s.lizs.dec", argv[1]); - - printf("inp = [%s]\n", inpFilename); - printf("lizard = [%s]\n", lizardFilename); - printf("dec = [%s]\n", decFilename); - - // compress - { - FILE* inpFp = fopen(inpFilename, "rb"); - FILE* outFp = fopen(lizardFilename, "wb"); - - test_compress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES); - - fclose(outFp); - fclose(inpFp); - } - - // decompress - { - FILE* inpFp = fopen(lizardFilename, "rb"); - FILE* outFp = fopen(decFilename, "wb"); - - test_decompress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES); - - fclose(outFp); - fclose(inpFp); - } - - // verify - { - FILE* inpFp = fopen(inpFilename, "rb"); - FILE* decFp = fopen(decFilename, "rb"); - - const int cmp = compare(inpFp, decFp); - if (0 == cmp) - printf("Verify : OK\n"); - else - printf("Verify : NG\n"); - - fclose(decFp); - fclose(inpFp); - } - - return 0; -} diff --git a/contrib/lizard/examples/blockStreaming_lineByLine.md b/contrib/lizard/examples/blockStreaming_lineByLine.md deleted file mode 100644 index ac56b879c90..00000000000 --- a/contrib/lizard/examples/blockStreaming_lineByLine.md +++ /dev/null @@ -1,122 +0,0 @@ -# Lizard streaming API Example : Line by Line Text Compression -by *Takayuki Matsuoka* - -`blockStreaming_lineByLine.c` is Lizard Straming API example which implements line by line incremental (de)compression. - -Please note the following restrictions : - - - Firstly, read "Lizard Streaming API Basics". - - This is relatively advanced application example. - - Output file is not compatible with lizard_frame and platform dependent. - - -## What's the point of this example ? - - - Line by line incremental (de)compression. - - Handle huge file in small amount of memory - - Generally better compression ratio than Block API - - Non-uniform block size - - -## How the compression works - -First of all, allocate "Ring Buffer" for input and Lizard compressed data buffer for output. - -``` -(1) - Ring Buffer - - +--------+ - | Line#1 | - +---+----+ - | - v - {Out#1} - - -(2) - Prefix Mode Dependency - +----+ - | | - v | - +--------+-+------+ - | Line#1 | Line#2 | - +--------+---+----+ - | - v - {Out#2} - - -(3) - Prefix Prefix - +----+ +----+ - | | | | - v | v | - +--------+-+------+-+------+ - | Line#1 | Line#2 | Line#3 | - +--------+--------+---+----+ - | - v - {Out#3} - - -(4) - External Dictionary Mode - +----+ +----+ - | | | | - v | v | - ------+--------+-+------+-+--------+ - | .... | Line#X | Line#X+1 | - ------+--------+--------+-----+----+ - ^ | - | v - | {Out#X+1} - | - Reset - - -(5) - Prefix - +-----+ - | | - v | - ------+--------+--------+----------+--+-------+ - | .... | Line#X | Line#X+1 | Line#X+2 | - ------+--------+--------+----------+-----+----+ - ^ | - | v - | {Out#X+2} - | - Reset -``` - -Next (see (1)), read first line to ringbuffer and compress it by `Lizard_compress_continue()`. -For the first time, Lizard doesn't know any previous dependencies, -so it just compress the line without dependencies and generates compressed line {Out#1} to Lizard compressed data buffer. -After that, write {Out#1} to the file and forward ringbuffer offset. - -Do the same things to second line (see (2)). -But in this time, Lizard can use dependency to Line#1 to improve compression ratio. -This dependency is called "Prefix mode". - -Eventually, we'll reach end of ringbuffer at Line#X (see (4)). -This time, we should reset ringbuffer offset. -After resetting, at Line#X+1 pointer is not adjacent, but Lizard still maintain its memory. -This is called "External Dictionary Mode". - -In Line#X+2 (see (5)), finally Lizard forget almost all memories but still remains Line#X+1. -This is the same situation as Line#2. - -Continue these procedure to the end of text file. - - -## How the decompression works - -Decompression will do reverse order. - - - Read compressed line from the file to buffer. - - Decompress it to the ringbuffer. - - Output decompressed plain text line to the file. - - Forward ringbuffer offset. If offset exceedes end of the ringbuffer, reset it. - -Continue these procedure to the end of the compressed file. diff --git a/contrib/lizard/examples/blockStreaming_ringBuffer.c b/contrib/lizard/examples/blockStreaming_ringBuffer.c deleted file mode 100644 index 011cd8b665b..00000000000 --- a/contrib/lizard/examples/blockStreaming_ringBuffer.c +++ /dev/null @@ -1,202 +0,0 @@ -// Lizard streaming API example : ring buffer -// Based on sample code from Takayuki Matsuoka - - -/************************************** - * Compiler Options - **************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS // for MSVC -# define snprintf sprintf_s -#endif -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif - - -/************************************** - * Includes - **************************************/ -#include -#include -#include -#include -#include "lizard_common.h" -#include "lizard_decompress.h" - - -enum { - MESSAGE_MAX_BYTES = 1024, - RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES, - DECODE_RING_BUFFER = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES // Intentionally larger, to test unsynchronized ring buffers -}; - - -size_t write_int32(FILE* fp, int32_t i) { - return fwrite(&i, sizeof(i), 1, fp); -} - -size_t write_bin(FILE* fp, const void* array, int arrayBytes) { - return fwrite(array, 1, arrayBytes, fp); -} - -size_t read_int32(FILE* fp, int32_t* i) { - return fread(i, sizeof(*i), 1, fp); -} - -size_t read_bin(FILE* fp, void* array, int arrayBytes) { - return fread(array, 1, arrayBytes, fp); -} - - -void test_compress(FILE* outFp, FILE* inpFp) -{ - Lizard_stream_t* lizardStream = Lizard_createStream_MinLevel(); - if (!lizardStream) return; - - static char inpBuf[RING_BUFFER_BYTES]; - int inpOffset = 0; - - for(;;) { - // Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer. - char* const inpPtr = &inpBuf[inpOffset]; - const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1; - const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); - if (0 == inpBytes) break; - - { - char cmpBuf[LIZARD_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; - const int cmpBytes = Lizard_compress_continue(lizardStream, inpPtr, cmpBuf, inpBytes, Lizard_compressBound(inpBytes)); - if(cmpBytes <= 0) break; - write_int32(outFp, cmpBytes); - write_bin(outFp, cmpBuf, cmpBytes); - - inpOffset += inpBytes; - - // Wraparound the ringbuffer offset - if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) inpOffset = 0; - } - } - - write_int32(outFp, 0); - Lizard_freeStream(lizardStream); -} - - -void test_decompress(FILE* outFp, FILE* inpFp) -{ - static char decBuf[DECODE_RING_BUFFER]; - int decOffset = 0; - Lizard_streamDecode_t lizardStreamDecode_body = { 0 }; - Lizard_streamDecode_t* lizardStreamDecode = &lizardStreamDecode_body; - - for(;;) { - int cmpBytes = 0; - char cmpBuf[LIZARD_COMPRESSBOUND(MESSAGE_MAX_BYTES)]; - - { - const size_t r0 = read_int32(inpFp, &cmpBytes); - if(r0 != 1 || cmpBytes <= 0) break; - - const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes); - if(r1 != (size_t) cmpBytes) break; - } - - { - char* const decPtr = &decBuf[decOffset]; - const int decBytes = Lizard_decompress_safe_continue( - lizardStreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES); - if(decBytes <= 0) break; - decOffset += decBytes; - write_bin(outFp, decPtr, decBytes); - - // Wraparound the ringbuffer offset - if(decOffset >= DECODE_RING_BUFFER - MESSAGE_MAX_BYTES) decOffset = 0; - } - } -} - - -int compare(FILE* f0, FILE* f1) -{ - int result = 0; - - while(0 == result) { - char b0[65536]; - char b1[65536]; - const size_t r0 = fread(b0, 1, sizeof(b0), f0); - const size_t r1 = fread(b1, 1, sizeof(b1), f1); - - result = (int) r0 - (int) r1; - - if(0 == r0 || 0 == r1) { - break; - } - if(0 == result) { - result = memcmp(b0, b1, r0); - } - } - - return result; -} - - -int main(int argc, char** argv) -{ - char inpFilename[256] = { 0 }; - char lizardFilename[256] = { 0 }; - char decFilename[256] = { 0 }; - - if(argc < 2) { - printf("Please specify input filename\n"); - return 0; - } - - snprintf(inpFilename, 256, "%s", argv[1]); - snprintf(lizardFilename, 256, "%s.lizs-%d", argv[1], 0); - snprintf(decFilename, 256, "%s.lizs-%d.dec", argv[1], 0); - - printf("inp = [%s]\n", inpFilename); - printf("lizard = [%s]\n", lizardFilename); - printf("dec = [%s]\n", decFilename); - - // compress - { - FILE* inpFp = fopen(inpFilename, "rb"); - FILE* outFp = fopen(lizardFilename, "wb"); - - test_compress(outFp, inpFp); - - fclose(outFp); - fclose(inpFp); - } - - // decompress - { - FILE* inpFp = fopen(lizardFilename, "rb"); - FILE* outFp = fopen(decFilename, "wb"); - - test_decompress(outFp, inpFp); - - fclose(outFp); - fclose(inpFp); - } - - // verify - { - FILE* inpFp = fopen(inpFilename, "rb"); - FILE* decFp = fopen(decFilename, "rb"); - - const int cmp = compare(inpFp, decFp); - if(0 == cmp) { - printf("Verify : OK\n"); - } else { - printf("Verify : NG\n"); - } - - fclose(decFp); - fclose(inpFp); - } - - return 0; -} diff --git a/contrib/lizard/examples/compress_functions.c b/contrib/lizard/examples/compress_functions.c deleted file mode 100644 index 36bc40e691a..00000000000 --- a/contrib/lizard/examples/compress_functions.c +++ /dev/null @@ -1,303 +0,0 @@ -/* - * compress_functions.c - * Copyright : Kyle Harper - * License : Follows same licensing as the lizard_compress.c/lizard_compress.h program at any given time. Currently, BSD 2. - * Description: A program to demonstrate the various compression functions involved in when using Lizard_compress_MinLevel(). The idea - * is to show how each step in the call stack can be used directly, if desired. There is also some benchmarking for - * each function to demonstrate the (probably lack of) performance difference when jumping the stack. - * (If you're new to lizard, please read simple_buffer.c to understand the fundamentals) - * - * The call stack (before theoretical compiler optimizations) for Lizard_compress_MinLevel is as follows: - * Lizard_compress_MinLevel - * Lizard_compress_fast - * Lizard_compress_extState_MinLevel - * Lizard_compress_generic - * - * Lizard_compress_MinLevel() - * This is the recommended function for compressing data. It will serve as the baseline for comparison. - * Lizard_compress_fast() - * Despite its name, it's not a "fast" version of compression. It simply decides if HEAPMODE is set and either - * allocates memory on the heap for a struct or creates the struct directly on the stack. Stack access is generally - * faster but this function itself isn't giving that advantage, it's just some logic for compile time. - * Lizard_compress_extState_MinLevel() - * This simply accepts all the pointers and values collected thus far and adds logic to determine how - * Lizard_compress_generic should be invoked; specifically: can the source fit into a single pass as determined by - * Lizard_64Klimit. - * Lizard_compress_generic() - * As the name suggests, this is the generic function that ultimately does most of the heavy lifting. Calling this - * directly can help avoid some test cases and branching which might be useful in some implementation-specific - * situations, but you really need to know what you're doing AND what you're asking lizard to do! You also need a - * wrapper function because this function isn't exposed with lizard_compress.h. - * - * The call stack for decompression functions is shallow. There are 2 options: - * Lizard_decompress_safe || Lizard_decompress_fast - * Lizard_decompress_generic - * - * Lizard_decompress_safe - * This is the recommended function for decompressing data. It is considered safe because the caller specifies - * both the size of the compresssed buffer to read as well as the maximum size of the output (decompressed) buffer - * instead of just the latter. - * Lizard_decompress_generic - * This is the generic function that both of the Lizard_decompress_* functions above end up calling. Calling this - * directly is not advised, period. Furthermore, it is a static inline function in lizard_compress.c, so there isn't a symbol - * exposed for anyone using lizard_compress.h to utilize. - * - * Special Note About Decompression: - * Using the Lizard_decompress_safe() function protects against malicious (user) input. - */ - -/* Since lizard compiles with c99 and not gnu/std99 we need to enable POSIX linking for time.h structs and functions. */ -#if __STDC_VERSION__ >= 199901L -#define _XOPEN_SOURCE 600 -#else -#define _XOPEN_SOURCE 500 -#endif -#define _POSIX_C_SOURCE 199309L - -/* Includes, for Power! */ -#include "lizard_compress.h" -#include "lizard_decompress.h" -#include /* for printf() */ -#include /* for exit() */ -#include /* for atoi() memcmp() */ -#include /* for uint_types */ -#include /* for PRIu64 */ -#include /* for clock_gettime() */ -#include /* for setlocale() */ - -/* We need to know what one billion is for clock timing. */ -#define BILLION 1000000000L - -/* Create a crude set of test IDs so we can switch on them later (Can't switch() on a char[] or char*). */ -#define ID__LIZARD_COMPRESS_DEFAULT 1 -#define ID__LIZARD_COMPRESS_GENERIC 4 -#define ID__LIZARD_DECOMPRESS_SAFE 5 - - - -/* - * Easy show-error-and-bail function. - */ -void run_screaming(const char *message, const int code) { - printf("%s\n", message); - exit(code); - return; -} - - -/* - * Centralize the usage function to keep main cleaner. - */ -void usage(const char *message) { - printf("Usage: ./argPerformanceTesting \n"); - run_screaming(message, 1); - return; -} - - - -/* - * Runs the benchmark for Lizard_compress_* based on function_id. - */ -uint64_t bench( - const char *known_good_dst, - const int function_id, - const int iterations, - const char *src, - char *dst, - const size_t src_size, - const size_t max_dst_size, - const size_t comp_size - ) { - uint64_t time_taken = 0; - int rv = 0; - const int warm_up = 5000; - struct timespec start, end; - Lizard_stream_t* state = Lizard_createStream_MinLevel(); - if (!state) return; - - // Select the right function to perform the benchmark on. We perform 5000 initial loops to warm the cache and ensure that dst - // remains matching to known_good_dst between successive calls. - switch(function_id) { - case ID__LIZARD_COMPRESS_DEFAULT: - printf("Starting benchmark for function: Lizard_compress_MinLevel()\n"); - for(int junk=0; junk 1) - iterations = atoi(argv[1]); - if (iterations < 1) - usage("Argument 1 (iterations) must be > 0."); - - // First we will create 2 sources (char *) of 2000 bytes each. One normal text, the other highly-compressible text. - const char *src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed luctus purus et risus vulputate, et mollis orci ullamcorper. Nulla facilisi. Fusce in ligula sed purus varius aliquet interdum vitae justo. Proin quis diam velit. Nulla varius iaculis auctor. Cras volutpat, justo eu dictum pulvinar, elit sem porttitor metus, et imperdiet metus sapien et ante. Nullam nisi nulla, ornare eu tristique eu, dignissim vitae diam. Nulla sagittis porta libero, a accumsan felis sagittis scelerisque. Integer laoreet eleifend congue. Etiam rhoncus leo vel dolor fermentum, quis luctus nisl iaculis. Praesent a erat sapien. Aliquam semper mi in lorem ultrices ultricies. Lorem ipsum dolor sit amet, consectetur adipiscing elit. In feugiat risus sed enim ultrices, at sodales nulla tristique. Maecenas eget pellentesque justo, sed pellentesque lectus. Fusce sagittis sit amet elit vel varius. Donec sed ligula nec ligula vulputate rutrum sed ut lectus. Etiam congue pharetra leo vitae cursus. Morbi enim ante, porttitor ut varius vel, tincidunt quis justo. Nunc iaculis, risus id ultrices semper, metus est efficitur ligula, vel posuere risus nunc eget purus. Ut lorem turpis, condimentum at sem sed, porta aliquam turpis. In ut sapien a nulla dictum tincidunt quis sit amet lorem. Fusce at est egestas, luctus neque eu, consectetur tortor. Phasellus eleifend ultricies nulla ac lobortis. Morbi maximus quam cursus vehicula iaculis. Maecenas cursus vel justo ut rutrum. Curabitur magna orci, dignissim eget dapibus vitae, finibus id lacus. Praesent rhoncus mattis augue vitae bibendum. Praesent porta mauris non ultrices fermentum. Quisque vulputate ipsum in sodales pulvinar. Aliquam nec mollis felis. Donec vitae augue pulvinar, congue nisl sed, pretium purus. Fusce lobortis mi ac neque scelerisque semper. Pellentesque vel est vitae magna aliquet aliquet. Nam non dolor. Nulla facilisi. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Morbi ac lacinia felis metus."; - const char *hc_src = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - // Set and derive sizes. Since we're using strings, use strlen() + 1 for \0. - const size_t src_size = strlen(src) + 1; - const size_t max_dst_size = Lizard_compressBound(src_size); - int bytes_returned = 0; - // Now build allocations for the data we'll be playing with. - char *dst = calloc(1, max_dst_size); - char *known_good_dst = calloc(1, max_dst_size); - char *known_good_hc_dst = calloc(1, max_dst_size); - if (dst == NULL || known_good_dst == NULL || known_good_hc_dst == NULL) - run_screaming("Couldn't allocate memory for the destination buffers. Sad :(", 1); - - // Create known-good buffers to verify our tests with other functions will produce the same results. - bytes_returned = Lizard_compress_MinLevel(src, known_good_dst, src_size, max_dst_size); - if (bytes_returned < 1) - run_screaming("Couldn't create a known-good destination buffer for comparison... this is bad.", 1); - const size_t src_comp_size = bytes_returned; - bytes_returned = Lizard_compress_MinLevel(hc_src, known_good_hc_dst, src_size, max_dst_size); - if (bytes_returned < 1) - run_screaming("Couldn't create a known-good (highly compressible) destination buffer for comparison... this is bad.", 1); - const size_t hc_src_comp_size = bytes_returned; - - - /* Lizard_compress_MinLevel() */ - // This is the default function so we don't need to demonstrate how to use it. See basics.c if you need more basal information. - - /* Lizard_compress_extState_MinLevel() */ - // Using this function directly requires that we build an Lizard_stream_t struct ourselves. We do NOT have to reset it ourselves. - memset(dst, 0, max_dst_size); - Lizard_stream_t* state = Lizard_createStream_MinLevel(); - if (!state) return; - bytes_returned = Lizard_compress_extState_MinLevel(state, src, dst, src_size, max_dst_size, 1); - if (bytes_returned < 1) - run_screaming("Failed to compress src using Lizard_compress_extState_MinLevel. echo $? for return code.", bytes_returned); - if (memcmp(dst, known_good_dst, bytes_returned) != 0) - run_screaming("According to memcmp(), the value we got in dst from Lizard_compress_extState_MinLevel doesn't match the known-good value. This is bad.", 1); - - /* Lizard_compress_generic */ - // When you can exactly control the inputs and options of your Lizard needs, you can use Lizard_compress_generic and fixed (const) - // values for the enum types such as dictionary and limitations. Any other direct-use is probably a bad idea. - // - // That said, the Lizard_compress_generic() function is 'static inline' and does not have a prototype in lizard_compress.h to expose a symbol - // for it. In other words: we can't access it directly. I don't want to submit a PR that modifies lizard_compress.c/h. Yann and others can - // do that if they feel it's worth expanding this example. - // - // I will, however, leave a skeleton of what would be required to use it directly: - /* - memset(dst, 0, max_dst_size); - // Lizard_stream_t state: is already declared above. We can reuse it BUT we have to reset the stream ourselves between each call. - Lizard_resetStream_MinLevel((Lizard_stream_t *)state); - // Since src size is small we know the following enums will be used: notLimited (0), byU16 (2), noDict (0), noDictIssue (0). - bytes_returned = Lizard_compress_generic(state, src, dst, src_size, max_dst_size, notLimited, byU16, noDict, noDictIssue, 1); - if (bytes_returned < 1) - run_screaming("Failed to compress src using Lizard_compress_generic. echo $? for return code.", bytes_returned); - if (memcmp(dst, known_good_dst, bytes_returned) != 0) - run_screaming("According to memcmp(), the value we got in dst from Lizard_compress_generic doesn't match the known-good value. This is bad.", 1); - */ - Lizard_freeStream(state); - - - /* Benchmarking */ - /* Now we'll run a few rudimentary benchmarks with each function to demonstrate differences in speed based on the function used. - * Remember, we cannot call Lizard_compress_generic() directly (yet) so it's disabled. - */ - // Suite A - Normal Compressibility - char *dst_d = calloc(1, src_size); - memset(dst, 0, max_dst_size); - printf("\nStarting suite A: Normal compressible text.\n"); - uint64_t time_taken__default = bench(known_good_dst, ID__LIZARD_COMPRESS_DEFAULT, iterations, src, dst, src_size, max_dst_size, src_comp_size); - //uint64_t time_taken__generic = bench(known_good_dst, ID__LIZARD_COMPRESS_GENERIC, iterations, src, dst, src_size, max_dst_size, src_comp_size); - uint64_t time_taken__decomp_safe = bench(src, ID__LIZARD_DECOMPRESS_SAFE, iterations, known_good_dst, dst_d, src_size, max_dst_size, src_comp_size); - // Suite B - Highly Compressible - memset(dst, 0, max_dst_size); - printf("\nStarting suite B: Highly compressible text.\n"); - uint64_t time_taken_hc__default = bench(known_good_hc_dst, ID__LIZARD_COMPRESS_DEFAULT, iterations, hc_src, dst, src_size, max_dst_size, hc_src_comp_size); - //uint64_t time_taken_hc__generic = bench(known_good_hc_dst, ID__LIZARD_COMPRESS_GENERIC, iterations, hc_src, dst, src_size, max_dst_size, hc_src_comp_size); - uint64_t time_taken_hc__decomp_safe = bench(hc_src, ID__LIZARD_DECOMPRESS_SAFE, iterations, known_good_hc_dst, dst_d, src_size, max_dst_size, hc_src_comp_size); - - // Report and leave. - setlocale(LC_ALL, ""); - const char *format = "|%-14s|%-30s|%'14.9f|%'16d|%'14d|%'13.2f%%|\n"; - const char *header_format = "|%-14s|%-30s|%14s|%16s|%14s|%14s|\n"; - const char *separator = "+--------------+------------------------------+--------------+----------------+--------------+--------------+\n"; - printf("\n"); - printf("%s", separator); - printf(header_format, "Source", "Function Benchmarked", "Total Seconds", "Iterations/sec", "ns/Iteration", "% of default"); - printf("%s", separator); - printf(format, "Normal Text", "Lizard_compress_MinLevel()", (double)time_taken__default / BILLION, (int)(iterations / ((double)time_taken__default /BILLION)), time_taken__default / iterations, (double)time_taken__default * 100 / time_taken__default); - printf(format, "Normal Text", "Lizard_compress_fast()", (double)time_taken__fast / BILLION, (int)(iterations / ((double)time_taken__fast /BILLION)), time_taken__fast / iterations, (double)time_taken__fast * 100 / time_taken__default); - printf(format, "Normal Text", "Lizard_compress_extState_MinLevel()", (double)time_taken__fast_extstate / BILLION, (int)(iterations / ((double)time_taken__fast_extstate /BILLION)), time_taken__fast_extstate / iterations, (double)time_taken__fast_extstate * 100 / time_taken__default); - //printf(format, "Normal Text", "Lizard_compress_generic()", (double)time_taken__generic / BILLION, (int)(iterations / ((double)time_taken__generic /BILLION)), time_taken__generic / iterations, (double)time_taken__generic * 100 / time_taken__default); - printf(format, "Normal Text", "Lizard_decompress_safe()", (double)time_taken__decomp_safe / BILLION, (int)(iterations / ((double)time_taken__decomp_safe /BILLION)), time_taken__decomp_safe / iterations, (double)time_taken__decomp_safe * 100 / time_taken__default); - printf(header_format, "", "", "", "", "", ""); - printf(format, "Compressible", "Lizard_compress_MinLevel()", (double)time_taken_hc__default / BILLION, (int)(iterations / ((double)time_taken_hc__default /BILLION)), time_taken_hc__default / iterations, (double)time_taken_hc__default * 100 / time_taken_hc__default); - printf(format, "Compressible", "Lizard_compress_fast()", (double)time_taken_hc__fast / BILLION, (int)(iterations / ((double)time_taken_hc__fast /BILLION)), time_taken_hc__fast / iterations, (double)time_taken_hc__fast * 100 / time_taken_hc__default); - printf(format, "Compressible", "Lizard_compress_extState_MinLevel()", (double)time_taken_hc__fast_extstate / BILLION, (int)(iterations / ((double)time_taken_hc__fast_extstate /BILLION)), time_taken_hc__fast_extstate / iterations, (double)time_taken_hc__fast_extstate * 100 / time_taken_hc__default); - //printf(format, "Compressible", "Lizard_compress_generic()", (double)time_taken_hc__generic / BILLION, (int)(iterations / ((double)time_taken_hc__generic /BILLION)), time_taken_hc__generic / iterations, (double)time_taken_hc__generic * 100 / time_taken_hc__default); - printf(format, "Compressible", "Lizard_decompress_safe()", (double)time_taken_hc__decomp_safe / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_safe /BILLION)), time_taken_hc__decomp_safe / iterations, (double)time_taken_hc__decomp_safe * 100 / time_taken_hc__default); - printf("%s", separator); - printf("\n"); - printf("All done, ran %d iterations per test.\n", iterations); - return 0; -} diff --git a/contrib/lizard/examples/frameCompress.c b/contrib/lizard/examples/frameCompress.c deleted file mode 100644 index 37ea0aa4fa8..00000000000 --- a/contrib/lizard/examples/frameCompress.c +++ /dev/null @@ -1,169 +0,0 @@ -// Lizardframe API example : compress a file -// Based on sample code from Zbigniew Jędrzejewski-Szmek - -#include -#include -#include -#include - -#include - -#define BUF_SIZE (16*1024) -#define LIZARD_HEADER_SIZE 19 -#define LIZARD_FOOTER_SIZE 4 - -static const LizardF_preferences_t lizard_preferences = { - { LizardF_max256KB, LizardF_blockLinked, LizardF_noContentChecksum, LizardF_frame, 0, { 0, 0 } }, - 0, /* compression level */ - 0, /* autoflush */ - { 0, 0, 0, 0 }, /* reserved, must be set to 0 */ -}; - -static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) { - LizardF_errorCode_t r; - LizardF_compressionContext_t ctx; - char *src, *buf = NULL; - size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size; - - r = LizardF_createCompressionContext(&ctx, LIZARDF_VERSION); - if (LizardF_isError(r)) { - printf("Failed to create context: error %zu", r); - return 1; - } - r = 1; - - src = malloc(BUF_SIZE); - if (!src) { - printf("Not enough memory"); - goto cleanup; - } - - frame_size = LizardF_compressBound(BUF_SIZE, &lizard_preferences); - size = frame_size + LIZARD_HEADER_SIZE + LIZARD_FOOTER_SIZE; - buf = malloc(size); - if (!buf) { - printf("Not enough memory"); - goto cleanup; - } - - n = offset = count_out = LizardF_compressBegin(ctx, buf, size, &lizard_preferences); - if (LizardF_isError(n)) { - printf("Failed to start compression: error %zu", n); - goto cleanup; - } - - printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n); - - for (;;) { - k = fread(src, 1, BUF_SIZE, in); - if (k == 0) - break; - count_in += k; - - n = LizardF_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL); - if (LizardF_isError(n)) { - printf("Compression failed: error %zu", n); - goto cleanup; - } - - offset += n; - count_out += n; - if (size - offset < frame_size + LIZARD_FOOTER_SIZE) { - printf("Writing %zu bytes\n", offset); - - k = fwrite(buf, 1, offset, out); - if (k < offset) { - if (ferror(out)) - printf("Write failed"); - else - printf("Short write"); - goto cleanup; - } - - offset = 0; - } - } - - n = LizardF_compressEnd(ctx, buf + offset, size - offset, NULL); - if (LizardF_isError(n)) { - printf("Failed to end compression: error %zu", n); - goto cleanup; - } - - offset += n; - count_out += n; - printf("Writing %zu bytes\n", offset); - - k = fwrite(buf, 1, offset, out); - if (k < offset) { - if (ferror(out)) - printf("Write failed"); - else - printf("Short write"); - goto cleanup; - } - - *size_in = count_in; - *size_out = count_out; - r = 0; - cleanup: - if (ctx) - LizardF_freeCompressionContext(ctx); - free(src); - free(buf); - return r; -} - -static int compress(const char *input, const char *output) { - char *tmp = NULL; - FILE *in = NULL, *out = NULL; - size_t size_in = 0, size_out = 0; - int r = 1; - - if (!output) { - size_t len = strlen(input); - - output = tmp = malloc(len + 5); - if (!tmp) { - printf("Not enough memory"); - return 1; - } - strcpy(tmp, input); - strcpy(tmp + len, ".liz"); - } - - in = fopen(input, "rb"); - if (!in) { - fprintf(stderr, "Failed to open input file %s: %s\n", input, strerror(errno)); - goto cleanup; - } - - out = fopen(output, "wb"); - if (!out) { - fprintf(stderr, "Failed to open output file %s: %s\n", output, strerror(errno)); - goto cleanup; - } - - r = compress_file(in, out, &size_in, &size_out); - if (r == 0) - printf("%s: %zu → %zu bytes, %.1f%%\n", - input, size_in, size_out, - (double)size_out / size_in * 100); - cleanup: - if (in) - fclose(in); - if (out) - fclose(out); - free(tmp); - return r; -} - - -int main(int argc, char **argv) { - if (argc < 2 || argc > 3) { - fprintf(stderr, "Syntax: %s \n", argv[0]); - return EXIT_FAILURE; - } - - return compress(argv[1], argv[2]); -} diff --git a/contrib/lizard/examples/printVersion.c b/contrib/lizard/examples/printVersion.c deleted file mode 100644 index 9818fdd98cf..00000000000 --- a/contrib/lizard/examples/printVersion.c +++ /dev/null @@ -1,13 +0,0 @@ -// Lizard trivial example : print Library version number -// Copyright : Takayuki Matsuoka & Yann Collet - - -#include -#include "lizard_compress.h" - -int main(int argc, char** argv) -{ - (void)argc; (void)argv; - printf("Hello World ! Lizard Library version = %d\n", Lizard_versionNumber()); - return 0; -} diff --git a/contrib/lizard/examples/simple_buffer.c b/contrib/lizard/examples/simple_buffer.c deleted file mode 100644 index 154d3b2cb76..00000000000 --- a/contrib/lizard/examples/simple_buffer.c +++ /dev/null @@ -1,91 +0,0 @@ -/* - * simple_buffer.c - * Copyright : Kyle Harper - * License : Follows same licensing as the lizard_compress.c/lizard_compress.h program at any given time. Currently, BSD 2. - * Description: Example program to demonstrate the basic usage of the compress/decompress functions within lizard_compress.c/lizard_compress.h. - * The functions you'll likely want are Lizard_compress_MinLevel and Lizard_decompress_safe. Both of these are documented in - * the lizard_compress.h header file; I recommend reading them. - */ - -/* Includes, for Power! */ -#include "lizard_compress.h" // This is all that is required to expose the prototypes for basic compression and decompression. -#include "lizard_decompress.h" -#include // For printf() -#include // For memcmp() -#include // For exit() - -/* - * Easy show-error-and-bail function. - */ -void run_screaming(const char *message, const int code) { - printf("%s\n", message); - exit(code); - return; -} - - -/* - * main - */ -int main(void) { - /* Introduction */ - // Below we will have a Compression and Decompression section to demonstrate. There are a few important notes before we start: - // 1) The return codes of Lizard_ functions are important. Read lizard_compress.h if you're unsure what a given code means. - // 2) Lizard uses char* pointers in all Lizard_ functions. This is baked into the API and probably not going to change. If your - // program uses pointers that are unsigned char*, void*, or otherwise different you may need to do some casting or set the - // right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign). - - /* Compression */ - // We'll store some text into a variable pointed to by *src to be compressed later. - const char *src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; - // The compression function needs to know how many bytes of exist. Since we're using a string, we can use strlen() + 1 (for \0). - const size_t src_size = strlen(src) + 1; - // Lizard provides a function that will tell you the maximum size of compressed output based on input data via Lizard_compressBound(). - const size_t max_dst_size = Lizard_compressBound(src_size); - // We will use that size for our destination boundary when allocating space. - char *compressed_data = malloc(max_dst_size); - if (compressed_data == NULL) - run_screaming("Failed to allocate memory for *compressed_data.", 1); - // That's all the information and preparation Lizard needs to compress *src into *compressed_data. Invoke Lizard_compress_MinLevel now - // with our size values and pointers to our memory locations. Save the return value for error checking. - int return_value = 0; - return_value = Lizard_compress_MinLevel(src, compressed_data, src_size, max_dst_size); - // Check return_value to determine what happened. - if (return_value < 0) - run_screaming("A negative result from Lizard_compress_MinLevel indicates a failure trying to compress the data. See exit code (echo $?) for value returned.", return_value); - if (return_value == 0) - run_screaming("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.", 1); - if (return_value > 0) - printf("We successfully compressed some data!\n"); - // Not only does a positive return_value mean success, the value returned == the number of bytes required. You can use this to - // realloc() *compress_data to free up memory, if desired. We'll do so just to demonstrate the concept. - const size_t compressed_data_size = return_value; - compressed_data = (char *)realloc(compressed_data, compressed_data_size); - if (compressed_data == NULL) - run_screaming("Failed to re-alloc memory for compressed_data. Sad :(", 1); - - /* Decompression */ - // Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite! We'll create a - // *new_src location of size src_size since we know that value. - char *new_src = malloc(src_size); - if (new_src == NULL) - run_screaming("Failed to allocate memory for *new_src.", 1); - // The Lizard_decompress_safe function needs to know where the compressed data is, how many bytes long it is, where the new_src - // memory location is, and how large the new_src (uncompressed) output will be. Again, save the return_value. - return_value = Lizard_decompress_safe(compressed_data, new_src, compressed_data_size, src_size); - if (return_value < 0) - run_screaming("A negative result from Lizard_decompress_fast indicates a failure trying to decompress the data. See exit code (echo $?) for value returned.", return_value); - if (return_value == 0) - run_screaming("I'm not sure this function can ever return 0. Documentation in lizard_compress.h doesn't indicate so.", 1); - if (return_value > 0) - printf("We successfully decompressed some data!\n"); - // Not only does a positive return value mean success, the value returned == the number of bytes read from the compressed_data - // stream. I'm not sure there's ever a time you'll need to know this in most cases... - - /* Validation */ - // We should be able to compare our original *src with our *new_src and be byte-for-byte identical. - if (memcmp(src, new_src, src_size) != 0) - run_screaming("Validation failed. *src and *new_src are not identical.", 1); - printf("Validation done. The string we ended up with is:\n%s\n", new_src); - return 0; -} diff --git a/contrib/lizard/examples/streaming_api_basics.md b/contrib/lizard/examples/streaming_api_basics.md deleted file mode 100644 index 85c06803455..00000000000 --- a/contrib/lizard/examples/streaming_api_basics.md +++ /dev/null @@ -1,87 +0,0 @@ -# Lizard Streaming API Basics -by *Takayuki Matsuoka* -## Lizard API sets - -Lizard has the following API sets : - - - "Auto Framing" API (lizard_frame.h) : - This is most recommended API for usual application. - It guarantees interoperability with other Lizard framing format compliant tools/libraries - such as Lizard command line utility, node-lizard, etc. - - "Block" API : This is recommended for simple purpose. - It compress single raw memory block to Lizard memory block and vice versa. - - "Streaming" API : This is designed for complex thing. - For example, compress huge stream data in restricted memory environment. - -Basically, you should use "Auto Framing" API. -But if you want to write advanced application, it's time to use Block or Streaming APIs. - - -## What is difference between Block and Streaming API ? - -Block API (de)compresses single contiguous memory block. -In other words, Lizard library find redundancy from single contiguous memory block. -Streaming API does same thing but (de)compress multiple adjacent contiguous memory block. -So Lizard library could find more redundancy than Block API. - -The following figure shows difference between API and block sizes. -In these figures, original data is splitted to 4KiBytes contiguous chunks. - -``` -Original Data - +---------------+---------------+----+----+----+ - | 4KiB Chunk A | 4KiB Chunk B | C | D |... | - +---------------+---------------+----+----+----+ - -Example (1) : Block API, 4KiB Block - +---------------+---------------+----+----+----+ - | 4KiB Chunk A | 4KiB Chunk B | C | D |... | - +---------------+---------------+----+----+----+ - | Block #1 | Block #2 | #3 | #4 |... | - +---------------+---------------+----+----+----+ - - (No Dependency) - - -Example (2) : Block API, 8KiB Block - +---------------+---------------+----+----+----+ - | 4KiB Chunk A | 4KiB Chunk B | C | D |... | - +---------------+---------------+----+----+----+ - | Block #1 |Block #2 |... | - +--------------------+----------+-------+-+----+ - ^ | ^ | - | | | | - +--------------+ +----+ - Internal Dependency Internal Dependency - - -Example (3) : Streaming API, 4KiB Block - +---------------+---------------+-----+----+----+ - | 4KiB Chunk A | 4KiB Chunk B | C | D |... | - +---------------+---------------+-----+----+----+ - | Block #1 | Block #2 | #3 | #4 |... | - +---------------+----+----------+-+---+-+--+----+ - ^ | ^ | ^ | - | | | | | | - +--------------+ +--------+ +---+ - Dependency Dependency Dependency -``` - - - In example (1), there is no dependency. - All blocks are compressed independently. - - In example (2), naturally 8KiBytes block has internal dependency. - But still block #1 and #2 are compressed independently. - - In example (3), block #2 has dependency to #1, - also #3 has dependency to #2 and #1, #4 has #3, #2 and #1, and so on. - -Here, we can observe difference between example (2) and (3). -In (2), there's no dependency between chunk B and C, but (3) has dependency between B and C. -This dependency improves compression ratio. - - -## Restriction of Streaming API - -For the efficiency, Streaming API doesn't keep mirror copy of dependent (de)compressed memory. -This means users should keep these dependent (de)compressed memory explicitly. -Usually, "Dependent memory" is previous adjacent contiguous memory up to 64KiBytes. -Lizard will not access further memories. diff --git a/contrib/lizard/lib/.gitignore b/contrib/lizard/lib/.gitignore deleted file mode 100644 index 6eedf69343a..00000000000 --- a/contrib/lizard/lib/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# make install artefact -liblizard.pc -test/ diff --git a/contrib/lizard/lib/LICENSE b/contrib/lizard/lib/LICENSE deleted file mode 100644 index b335b0bafcf..00000000000 --- a/contrib/lizard/lib/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -Lizard Library -Copyright (C) 2011-2016, Yann Collet. -Copyright (C) 2016-2017, Przemyslaw Skibinski -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/lizard/lib/README.md b/contrib/lizard/lib/README.md deleted file mode 100644 index 2775b984d63..00000000000 --- a/contrib/lizard/lib/README.md +++ /dev/null @@ -1,76 +0,0 @@ -Lizard - Library Files -================================ - -The __lib__ directory contains several directories. -Depending on target use case, it's enough to include only files from relevant directories. - - -#### API - -Lizard stable API is exposed within [lizard_compress.h](lizard_compress.h) and [lizard_decompress.h](lizard_decompress.h), -at the root of `lib` directory. - - -#### Compatibility issues - -The raw Lizard block compression format is detailed within [lizard_Block_format]. -To compress an arbitrarily long file or data stream, multiple blocks are required. -Organizing these blocks and providing a common header format to handle their content -is the purpose of the Frame format, defined in [lizard_Frame_format]. -`lizard` command line utility produces files or streams compatible with the Frame format. -(_Advanced stuff_ : It's possible to hide xxhash symbols into a local namespace. -This is what `liblizard` does, to avoid symbol duplication -in case a user program would link to several libraries containing xxhash symbols.) - -[lizard_Block_format]: ../doc/lizard_Block_format.md -[lizard_Frame_format]: ../doc/lizard_Frame_format.md - - -#### Various Lizard builds - -Files `lizard_common.h`, `lizard_compress*`, `lizard_parser_*.h`, `lizard_decompress*`, and `entropy\mem.h` are required in all circumstances. - -To compile: -- Lizard_raw only with levels 10...29 : use the `-DLIZARD_NO_HUFFMAN` compiler flag -- Lizard_raw with levels 10...49 : include also all files from `entropy` directory -- Lizard_frame with levels 10...49 : `lizard_frame*` and all files from `entropy` and `xxhash` directories - - -#### Advanced API - -A more complex `lizard_frame_static.h` is also provided. -It contains definitions which are not guaranteed to remain stable within future versions. -It must be used with static linking ***only***. - - -#### Using MinGW+MSYS to create DLL - -DLL can be created using MinGW+MSYS with the `make liblizard` command. -This command creates `dll\liblizard.dll` and the import library `dll\liblizard.lib`. -The import library is only required with Visual C++. -The header files `lizard.h`, `lizardhc.h`, `lizard_frame.h` and the dynamic library -`dll\liblizard.dll` are required to compile a project using gcc/MinGW. -The dynamic library has to be added to linking options. -It means that if a project that uses Lizard consists of a single `test-dll.c` -file it should be compiled with `liblizard.lib`. For example: -``` - gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblizard.dll -``` -The compiled executable will require Lizard DLL which is available at `dll\liblizard.dll`. - - -#### Miscellaneous - -Other files present in the directory are not source code. There are : - - - LICENSE : contains the BSD license text - - Makefile : script to compile or install lizard library (static or dynamic) - - liblizard.pc.in : for pkg-config (make install) - - README.md : this file - - -#### License - -All source material within __lib__ directory are BSD 2-Clause licensed. -See [LICENSE](LICENSE) for details. -The license is also repeated at the top of each source file. diff --git a/contrib/lizard/lib/dll/liblizard.def b/contrib/lizard/lib/dll/liblizard.def deleted file mode 100644 index 73ebc44570a..00000000000 --- a/contrib/lizard/lib/dll/liblizard.def +++ /dev/null @@ -1,19 +0,0 @@ -LIBRARY liblizard.dll -EXPORTS - Lizard_compress - Lizard_compressBound - Lizard_compress_continue - Lizard_compress_extState - Lizard_createStream - Lizard_createStreamDecode - Lizard_decompress_safe - Lizard_decompress_safe_continue - Lizard_decompress_safe_partial - Lizard_decompress_safe_usingDict - Lizard_freeStream - Lizard_freeStreamDecode - Lizard_loadDict - Lizard_resetStream - Lizard_saveDict - Lizard_setStreamDecode - Lizard_sizeofState diff --git a/contrib/lizard/lib/entropy/README.md b/contrib/lizard/lib/entropy/README.md deleted file mode 100644 index 5b18f20b063..00000000000 --- a/contrib/lizard/lib/entropy/README.md +++ /dev/null @@ -1,38 +0,0 @@ -New Generation Entropy library -============================== - -The __lib__ directory contains several files, but you don't necessarily want them all. -Here is a detailed list, to help you decide which one you need : - - -#### Compulsory files - -These files are required in all circumstances : -- __error_public.h__ : error list as enum -- __error_private.h__ : error management -- __mem.h__ : low level memory access routines -- __bitstream.h__ : generic read/write bitstream common to all entropy codecs -- __entropy_common.c__ : common functions needed for both compression and decompression - - -#### Finite State Entropy - -This is the base codec required by other ones. -It implements a tANS variant, similar to arithmetic in compression performance, but much faster. Compression and decompression can be compiled independently. -- __fse.h__ : exposes interfaces -- __fse_compress.c__ : implements compression codec -- __fse_decompress.c__ : implements decompression codec - - -#### FSE 16-bits symbols version - -This codec is able to encode alphabets of size > 256, using 2 bytes per symbol. It requires the base FSE codec to compile properly. Compression and decompression are merged in the same file. -- __fseU16.c__ implements the codec, while __fseU16.h__ exposes its interfaces. - - -#### Huffman codec - -This is the fast huffman codec. It requires the base FSE codec to compress its headers. Compression and decompression can be compiled independently. -- __huf.h__ : exposes interfaces. -- __huf_compress.c__ : implements compression codec -- __huf_decompress.c__ : implements decompression codec diff --git a/contrib/lizard/lib/entropy/bitstream.h b/contrib/lizard/lib/entropy/bitstream.h deleted file mode 100644 index 849a9e6018a..00000000000 --- a/contrib/lizard/lib/entropy/bitstream.h +++ /dev/null @@ -1,414 +0,0 @@ -/* ****************************************************************** - bitstream - Part of FSE library - header file (to include) - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef BITSTREAM_H_MODULE -#define BITSTREAM_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* -* This API consists of small unitary functions, which must be inlined for best performance. -* Since link-time-optimization is not available for all compilers, -* these functions are defined into a .h to be included. -*/ - -/*-**************************************** -* Dependencies -******************************************/ -#include "mem.h" /* unaligned access routines */ -#include "error_private.h" /* error codes and messages */ - - -/*========================================= -* Target specific -=========================================*/ -#if defined(__BMI__) && defined(__GNUC__) -# include /* support for bextr (experimental) */ -#endif - - -/*-****************************************** -* bitStream encoding API (write forward) -********************************************/ -/* bitStream can mix input from multiple sources. -* A critical property of these streams is that they encode and decode in **reverse** direction. -* So the first bit sequence you add will be the last to be read, like a LIFO stack. -*/ -typedef struct -{ - size_t bitContainer; - int bitPos; - char* startPtr; - char* ptr; - char* endPtr; -} BIT_CStream_t; - -MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); -MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); -MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); -MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); - -/* Start with initCStream, providing the size of buffer to write into. -* bitStream will never write outside of this buffer. -* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. -* -* bits are first added to a local register. -* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. -* Writing data into memory is an explicit operation, performed by the flushBits function. -* Hence keep track how many bits are potentially stored into local register to avoid register overflow. -* After a flushBits, a maximum of 7 bits might still be stored into local register. -* -* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. -* -* Last operation is to close the bitStream. -* The function returns the final size of CStream in bytes. -* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) -*/ - - -/*-******************************************** -* bitStream decoding API (read backward) -**********************************************/ -typedef struct -{ - size_t bitContainer; - unsigned bitsConsumed; - const char* ptr; - const char* start; -} BIT_DStream_t; - -typedef enum { BIT_DStream_unfinished = 0, - BIT_DStream_endOfBuffer = 1, - BIT_DStream_completed = 2, - BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ - /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ - -MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); -MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); -MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); -MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); - - -/* Start by invoking BIT_initDStream(). -* A chunk of the bitStream is then stored into a local register. -* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). -* You can then retrieve bitFields stored into the local register, **in reverse order**. -* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. -* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. -* Otherwise, it can be less than that, so proceed accordingly. -* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). -*/ - - -/*-**************************************** -* unsafe API -******************************************/ -MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); -/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ - -MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); -/* unsafe version; does not check buffer overflow */ - -MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); -/* faster, but works only if nbBits >= 1 */ - - - -/*-************************************************************** -* Internal functions -****************************************************************/ -MEM_STATIC unsigned BIT_highbit32 (register U32 val) -{ -# if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - _BitScanReverse ( &r, val ); - return (unsigned) r; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); -# else /* Software version */ - static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; -# endif -} - -/*===== Local Constants =====*/ -static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ - - -/*-************************************************************** -* bitStream encoding -****************************************************************/ -/*! BIT_initCStream() : - * `dstCapacity` must be > sizeof(void*) - * @return : 0 if success, - otherwise an error code (can be tested using ERR_isError() ) */ -MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity) -{ - bitC->bitContainer = 0; - bitC->bitPos = 0; - bitC->startPtr = (char*)startPtr; - bitC->ptr = bitC->startPtr; - bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr); - if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall); - return 0; -} - -/*! BIT_addBits() : - can add up to 26 bits into `bitC`. - Does not check for register overflow ! */ -MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) -{ - bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; - bitC->bitPos += nbBits; -} - -/*! BIT_addBitsFast() : - * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ -MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits) -{ - bitC->bitContainer |= value << bitC->bitPos; - bitC->bitPos += nbBits; -} - -/*! BIT_flushBitsFast() : - * unsafe version; does not check buffer overflow */ -MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) -{ - size_t const nbBytes = bitC->bitPos >> 3; - MEM_writeLEST(bitC->ptr, bitC->bitContainer); - bitC->ptr += nbBytes; - bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ -} - -/*! BIT_flushBits() : - * safe version; check for buffer overflow, and prevents it. - * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */ -MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) -{ - size_t const nbBytes = bitC->bitPos >> 3; - MEM_writeLEST(bitC->ptr, bitC->bitContainer); - bitC->ptr += nbBytes; - if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; - bitC->bitPos &= 7; - bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ -} - -/*! BIT_closeCStream() : - * @return : size of CStream, in bytes, - or 0 if it could not fit into dstBuffer */ -MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) -{ - BIT_addBitsFast(bitC, 1, 1); /* endMark */ - BIT_flushBits(bitC); - - if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */ - - return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); -} - - -/*-******************************************************** -* bitStream decoding -**********************************************************/ -/*! BIT_initDStream() : -* Initialize a BIT_DStream_t. -* `bitD` : a pointer to an already allocated BIT_DStream_t structure. -* `srcSize` must be the *exact* size of the bitStream, in bytes. -* @return : size of stream (== srcSize) or an errorCode if a problem is detected -*/ -MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) -{ - if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } - - if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ - bitD->start = (const char*)srcBuffer; - bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); - bitD->bitContainer = MEM_readLEST(bitD->ptr); - { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ - if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } - } else { - bitD->start = (const char*)srcBuffer; - bitD->ptr = bitD->start; - bitD->bitContainer = *(const BYTE*)(bitD->start); - switch(srcSize) - { - case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); // fallthrough - case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); // fallthrough - case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); // fallthrough - case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; // fallthrough - case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; // fallthrough - case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; // fallthrough - default:; - } - { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; - if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } - bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; - } - - return srcSize; -} - -MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) -{ - return bitContainer >> start; -} - -MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) -{ -#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008 /* experimental */ -# if defined(__x86_64__) - if (sizeof(bitContainer)==8) - return _bextr_u64(bitContainer, start, nbBits); - else -# endif - return _bextr_u32(bitContainer, start, nbBits); -#else - return (bitContainer >> start) & BIT_mask[nbBits]; -#endif -} - -MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) -{ - return bitContainer & BIT_mask[nbBits]; -} - -/*! BIT_lookBits() : - * Provides next n bits from local register. - * local register is not modified. - * On 32-bits, maxNbBits==24. - * On 64-bits, maxNbBits==56. - * @return : value extracted - */ - MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) -{ -#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ - return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); -#else - U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; - return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); -#endif -} - -/*! BIT_lookBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) -{ - U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; - return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); -} - -MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) -{ - bitD->bitsConsumed += nbBits; -} - -/*! BIT_readBits() : - * Read (consume) next n bits from local register and update. - * Pay attention to not read more than nbBits contained into local register. - * @return : extracted value. - */ -MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) -{ - size_t const value = BIT_lookBits(bitD, nbBits); - BIT_skipBits(bitD, nbBits); - return value; -} - -/*! BIT_readBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) -{ - size_t const value = BIT_lookBitsFast(bitD, nbBits); - BIT_skipBits(bitD, nbBits); - return value; -} - -/*! BIT_reloadDStream() : -* Refill `bitD` from buffer previously set in BIT_initDStream() . -* This function is safe, it guarantees it will not read beyond src buffer. -* @return : status of `BIT_DStream_t` internal register. - if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ -MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) -{ - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ - return BIT_DStream_overflow; - - if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { - bitD->ptr -= bitD->bitsConsumed >> 3; - bitD->bitsConsumed &= 7; - bitD->bitContainer = MEM_readLEST(bitD->ptr); - return BIT_DStream_unfinished; - } - if (bitD->ptr == bitD->start) { - if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; - return BIT_DStream_completed; - } - { U32 nbBytes = bitD->bitsConsumed >> 3; - BIT_DStream_status result = BIT_DStream_unfinished; - if (bitD->ptr - nbBytes < bitD->start) { - nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ - result = BIT_DStream_endOfBuffer; - } - bitD->ptr -= nbBytes; - bitD->bitsConsumed -= nbBytes*8; - bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ - return result; - } -} - -/*! BIT_endOfDStream() : -* @return Tells if DStream has exactly reached its end (all bits consumed). -*/ -MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) -{ - return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); -} - -#if defined (__cplusplus) -} -#endif - -#endif /* BITSTREAM_H_MODULE */ diff --git a/contrib/lizard/lib/entropy/entropy_common.c b/contrib/lizard/lib/entropy/entropy_common.c deleted file mode 100644 index 72bc398da38..00000000000 --- a/contrib/lizard/lib/entropy/entropy_common.c +++ /dev/null @@ -1,231 +0,0 @@ -/* - Common functions of New Generation Entropy library - Copyright (C) 2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*************************************************************************** */ - -/* ************************************* -* Dependencies -***************************************/ -#include "mem.h" -#include "error_private.h" /* ERR_*, ERROR */ -#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ -#include "fse.h" -#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ -#include "huf.h" - - -/*-**************************************** -* Version -******************************************/ -unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } - - -/*-**************************************** -* FSE Error Management -******************************************/ -unsigned FSE_isError(size_t code) { return ERR_isError(code); } - -const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } - - -/* ************************************************************** -* HUF Error Management -****************************************************************/ -unsigned HUF_isError(size_t code) { return ERR_isError(code); } - -const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } - - -/*-************************************************************** -* FSE NCount encoding-decoding -****************************************************************/ -size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, - const void* headerBuffer, size_t hbSize) -{ - const BYTE* const istart = (const BYTE*) headerBuffer; - const BYTE* const iend = istart + hbSize; - const BYTE* ip = istart; - int nbBits; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - if (hbSize < 4) return ERROR(srcSize_wrong); - bitStream = MEM_readLE32(ip); - nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ - if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); - bitStream >>= 4; - bitCount = 4; - *tableLogPtr = nbBits; - remaining = (1<1) & (charnum<=*maxSVPtr)) { - if (previous0) { - unsigned n0 = charnum; - while ((bitStream & 0xFFFF) == 0xFFFF) { - n0 += 24; - if (ip < iend-5) { - ip += 2; - bitStream = MEM_readLE32(ip) >> bitCount; - } else { - bitStream >>= 16; - bitCount += 16; - } } - while ((bitStream & 3) == 3) { - n0 += 3; - bitStream >>= 2; - bitCount += 2; - } - n0 += bitStream & 3; - bitCount += 2; - if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); - while (charnum < n0) normalizedCounter[charnum++] = 0; - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - bitStream = MEM_readLE32(ip) >> bitCount; - } else { - bitStream >>= 2; - } } - { int const max = (2*threshold-1) - remaining; - int count; - - if ((bitStream & (threshold-1)) < (U32)max) { - count = bitStream & (threshold-1); - bitCount += nbBits-1; - } else { - count = bitStream & (2*threshold-1); - if (count >= threshold) count -= max; - bitCount += nbBits; - } - - count--; /* extra accuracy */ - remaining -= count < 0 ? -count : count; /* -1 means +1 */ - normalizedCounter[charnum++] = (short)count; - previous0 = !count; - while (remaining < threshold) { - nbBits--; - threshold >>= 1; - } - - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - } else { - bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } - bitStream = MEM_readLE32(ip) >> (bitCount & 31); - } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ - if (remaining != 1) return ERROR(corruption_detected); - if (bitCount > 32) return ERROR(corruption_detected); - *maxSVPtr = charnum-1; - - ip += (bitCount+7)>>3; - return ip-istart; -} - - -/*! HUF_readStats() : - Read compact Huffman tree, saved by HUF_writeCTable(). - `huffWeight` is destination buffer. - `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. - @return : size read from `src` , or an error Code . - Note : Needed by HUF_readCTable() and HUF_readDTableX?() . -*/ -size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, - U32* nbSymbolsPtr, U32* tableLogPtr, - const void* src, size_t srcSize) -{ - U32 weightTotal; - const BYTE* ip = (const BYTE*) src; - size_t iSize; - size_t oSize; - - if (!srcSize) return ERROR(srcSize_wrong); - iSize = ip[0]; - /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ - - if (iSize >= 128) { /* special header */ - oSize = iSize - 127; - iSize = ((oSize+1)/2); - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - if (oSize >= hwSize) return ERROR(corruption_detected); - ip += 1; - { U32 n; - for (n=0; n> 4; - huffWeight[n+1] = ip[n/2] & 15; - } } } - else { /* header compressed with FSE (normal case) */ - FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */ - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */ - if (FSE_isError(oSize)) return oSize; - } - - /* collect weight stats */ - memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); - weightTotal = 0; - { U32 n; for (n=0; n= HUF_TABLELOG_MAX) return ERROR(corruption_detected); - rankStats[huffWeight[n]]++; - weightTotal += (1 << huffWeight[n]) >> 1; - } } - if (weightTotal == 0) return ERROR(corruption_detected); - - /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BIT_highbit32(weightTotal) + 1; - if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); - *tableLogPtr = tableLog; - /* determine last weight */ - { U32 const total = 1 << tableLog; - U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; - if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ - huffWeight[oSize] = (BYTE)lastWeight; - rankStats[lastWeight]++; - } } - - /* check tree construction validity */ - if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ - - /* results */ - *nbSymbolsPtr = (U32)(oSize+1); - return iSize+1; -} diff --git a/contrib/lizard/lib/entropy/error_private.h b/contrib/lizard/lib/entropy/error_private.h deleted file mode 100644 index 1340c16bf25..00000000000 --- a/contrib/lizard/lib/entropy/error_private.h +++ /dev/null @@ -1,115 +0,0 @@ -/* ****************************************************************** - Error codes and messages - Copyright (C) 2013-2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ -/* Note : this module is expected to remain private, do not expose it */ - -#ifndef ERROR_H_MODULE -#define ERROR_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* Dependencies -******************************************/ -#include /* size_t */ -#include "error_public.h" /* enum list */ - - -/* **************************************** -* Compiler-specific -******************************************/ -#if defined(__GNUC__) -# define ERR_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define ERR_STATIC static inline -#elif defined(_MSC_VER) -# define ERR_STATIC static __inline -#else -# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - - -/*-**************************************** -* Customization (error_public.h) -******************************************/ -typedef FSE_ErrorCode ERR_enum; -#define PREFIX(name) FSE_error_##name - - -/*-**************************************** -* Error codes handling -******************************************/ -#ifdef ERROR -# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ -#endif -#define ERROR(name) ((size_t)-PREFIX(name)) - -ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } - -ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } - - -/*-**************************************** -* Error Strings -******************************************/ - -ERR_STATIC const char* ERR_getErrorString(ERR_enum code) -{ - static const char* notErrorCode = "Unspecified error code"; - switch( code ) - { - case PREFIX(no_error): return "No error detected"; - case PREFIX(GENERIC): return "Error (generic)"; - case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; - case PREFIX(corruption_detected): return "Corrupted block detected"; - case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; - case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; - case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; - case PREFIX(maxCode): - default: return notErrorCode; - } -} - -ERR_STATIC const char* ERR_getErrorName(size_t code) -{ - return ERR_getErrorString(ERR_getErrorCode(code)); -} - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_H_MODULE */ diff --git a/contrib/lizard/lib/entropy/error_public.h b/contrib/lizard/lib/entropy/error_public.h deleted file mode 100644 index a8524719889..00000000000 --- a/contrib/lizard/lib/entropy/error_public.h +++ /dev/null @@ -1,64 +0,0 @@ -/* ****************************************************************** - Error codes list - Copyright (C) 2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ -#ifndef ERROR_PUBLIC_H_MODULE -#define ERROR_PUBLIC_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* error codes list -******************************************/ -typedef enum { - FSE_error_no_error, - FSE_error_GENERIC, - FSE_error_dstSize_tooSmall, - FSE_error_srcSize_wrong, - FSE_error_corruption_detected, - FSE_error_tableLog_tooLarge, - FSE_error_maxSymbolValue_tooLarge, - FSE_error_maxSymbolValue_tooSmall, - FSE_error_maxCode -} FSE_ErrorCode; - -/* note : compare with size_t function results using FSE_getError() */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_PUBLIC_H_MODULE */ diff --git a/contrib/lizard/lib/entropy/fse.h b/contrib/lizard/lib/entropy/fse.h deleted file mode 100644 index baac3903267..00000000000 --- a/contrib/lizard/lib/entropy/fse.h +++ /dev/null @@ -1,694 +0,0 @@ -/* ****************************************************************** - FSE : Finite State Entropy codec - Public Prototypes declaration - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef FSE_H -#define FSE_H - -#if defined (__cplusplus) -extern "C" { -#endif - - -/*-***************************************** -* Dependencies -******************************************/ -#include /* size_t, ptrdiff_t */ - - -/*-***************************************** -* FSE_PUBLIC_API : control library symbols visibility -******************************************/ -#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) -# define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) -#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ -# define FSE_PUBLIC_API __declspec(dllexport) -#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) -# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define FSE_PUBLIC_API -#endif - -/*------ Version ------*/ -#define FSE_VERSION_MAJOR 0 -#define FSE_VERSION_MINOR 9 -#define FSE_VERSION_RELEASE 0 - -#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE -#define FSE_QUOTE(str) #str -#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) -#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) - -#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) -FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ - -/*-**************************************** -* FSE simple functions -******************************************/ -/*! FSE_compress() : - Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. - 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). - @return : size of compressed data (<= dstCapacity). - Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. - if FSE_isError(return), compression failed (more details using FSE_getErrorName()) -*/ -FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/*! FSE_decompress(): - Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', - into already allocated destination buffer 'dst', of size 'dstCapacity'. - @return : size of regenerated data (<= maxDstSize), - or an error code, which can be tested using FSE_isError() . - - ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! - Why ? : making this distinction requires a header. - Header management is intentionally delegated to the user layer, which can better manage special cases. -*/ -FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, - const void* cSrc, size_t cSrcSize); - - -/*-***************************************** -* Tool functions -******************************************/ -FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ - -/* Error Management */ -FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ -FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ - - -/*-***************************************** -* FSE advanced functions -******************************************/ -/*! FSE_compress2() : - Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' - Both parameters can be defined as '0' to mean : use default value - @return : size of compressed data - Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. - if FSE_isError(return), it's an error code. -*/ -FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); - - -/*-***************************************** -* FSE detailed API -******************************************/ -/*! -FSE_compress() does the following: -1. count symbol occurrence from source[] into table count[] -2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) -3. save normalized counters to memory buffer using writeNCount() -4. build encoding table 'CTable' from normalized counters -5. encode the data stream using encoding table 'CTable' - -FSE_decompress() does the following: -1. read normalized counters with readNCount() -2. build decoding table 'DTable' from normalized counters -3. decode the data stream using decoding table 'DTable' - -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and provide normalized distribution using external method. -*/ - -/* *** COMPRESSION *** */ - -/*! FSE_count(): - Provides the precise count of each byte within a table 'count'. - 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). - *maxSymbolValuePtr will be updated if detected smaller than initial value. - @return : the count of the most frequent symbol (which is not identified). - if return == srcSize, there is only one symbol. - Can also return an error code, which can be tested with FSE_isError(). */ -FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); - -/*! FSE_optimalTableLog(): - dynamically downsize 'tableLog' when conditions are met. - It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. - @return : recommended tableLog (necessarily <= 'maxTableLog') */ -FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); - -/*! FSE_normalizeCount(): - normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) - 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). - @return : tableLog, - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); - -/*! FSE_NCountWriteBound(): - Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. - Typically useful for allocation purpose. */ -FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); - -/*! FSE_writeNCount(): - Compactly save 'normalizedCounter' into 'buffer'. - @return : size of the compressed table, - or an errorCode, which can be tested using FSE_isError(). */ -FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - - -/*! Constructor and Destructor of FSE_CTable. - Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ -typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ -FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue); -FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); - -/*! FSE_buildCTable(): - Builds `ct`, which must be already allocated, using FSE_createCTable(). - @return : 0, or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! FSE_compress_usingCTable(): - Compress `src` using `ct` into `dst` which must be already allocated. - @return : size of compressed data (<= `dstCapacity`), - or 0 if compressed data could not fit into `dst`, - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); - -/*! -Tutorial : ----------- -The first step is to count all symbols. FSE_count() does this job very fast. -Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. -'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] -maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) -FSE_count() will return the number of occurrence of the most frequent symbol. -This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. -If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). - -The next step is to normalize the frequencies. -FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. -It also guarantees a minimum of 1 to any Symbol with frequency >= 1. -You can use 'tableLog'==0 to mean "use default tableLog value". -If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), -which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). - -The result of FSE_normalizeCount() will be saved into a table, -called 'normalizedCounter', which is a table of signed short. -'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. -The return value is tableLog if everything proceeded as expected. -It is 0 if there is a single symbol within distribution. -If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). - -'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). -'buffer' must be already allocated. -For guaranteed success, buffer size must be at least FSE_headerBound(). -The result of the function is the number of bytes written into 'buffer'. -If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). - -'normalizedCounter' can then be used to create the compression table 'CTable'. -The space required by 'CTable' must be already allocated, using FSE_createCTable(). -You can then use FSE_buildCTable() to fill 'CTable'. -If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). - -'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). -Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' -The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. -If it returns '0', compressed data could not fit into 'dst'. -If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). -*/ - - -/* *** DECOMPRESSION *** */ - -/*! FSE_readNCount(): - Read compactly saved 'normalizedCounter' from 'rBuffer'. - @return : size read from 'rBuffer', - or an errorCode, which can be tested using FSE_isError(). - maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ -FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); - -/*! Constructor and Destructor of FSE_DTable. - Note that its size depends on 'tableLog' */ -typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ -FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); -FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); - -/*! FSE_buildDTable(): - Builds 'dt', which must be already allocated, using FSE_createDTable(). - return : 0, or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! FSE_decompress_usingDTable(): - Decompress compressed source `cSrc` of size `cSrcSize` using `dt` - into `dst` which must be already allocated. - @return : size of regenerated data (necessarily <= `dstCapacity`), - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); - -/*! -Tutorial : ----------- -(Note : these functions only decompress FSE-compressed blocks. - If block is uncompressed, use memcpy() instead - If block is a single repeated byte, use memset() instead ) - -The first step is to obtain the normalized frequencies of symbols. -This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). -'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. -In practice, that means it's necessary to know 'maxSymbolValue' beforehand, -or size the table to handle worst case situations (typically 256). -FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. -The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. -Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. -If there is an error, the function will return an error code, which can be tested using FSE_isError(). - -The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. -This is performed by the function FSE_buildDTable(). -The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). -If there is an error, the function will return an error code, which can be tested using FSE_isError(). - -`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). -`cSrcSize` must be strictly correct, otherwise decompression will fail. -FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). -If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) -*/ - - -#ifdef FSE_STATIC_LINKING_ONLY - -/* *** Dependency *** */ -#include "bitstream.h" - - -/* ***************************************** -* Static allocation -*******************************************/ -/* FSE buffer bounds */ -#define FSE_NCOUNTBOUND 512 -#define FSE_BLOCKBOUND(size) (size + (size>>7)) -#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ - -/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ -#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) -#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<= `1024` unsigned - */ -size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, unsigned* workSpace); - -/** FSE_countFast() : - * same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr - */ -size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); - -/* FSE_countFast_wksp() : - * Same as FSE_countFast(), but using an externally provided scratch buffer. - * `workSpace` must be a table of minimum `1024` unsigned - */ -size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace); - -/*! FSE_count_simple - * Same as FSE_countFast(), but does not use any additional memory (not even on stack). - * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`). -*/ -size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); - - - -unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); -/**< same as FSE_optimalTableLog(), which used `minus==2` */ - -/* FSE_compress_wksp() : - * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). - * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. - */ -#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + (1<<((maxTableLog>2)?(maxTableLog-2):0)) ) -size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); - -size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); -/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ - -size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); -/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ - -/* FSE_buildCTable_wksp() : - * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). - * `wkspSize` must be >= `(1<= BIT_DStream_completed - -When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. -Checking if DStream has reached its end is performed by : - BIT_endOfDStream(&DStream); -Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. - FSE_endOfDState(&DState); -*/ - - -/* ***************************************** -* FSE unsafe API -*******************************************/ -static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); -/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ - - -/* ***************************************** -* Implementation of inlined functions -*******************************************/ -typedef struct { - int deltaFindState; - U32 deltaNbBits; -} FSE_symbolCompressionTransform; /* total 8 bytes */ - -MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) -{ - const void* ptr = ct; - const U16* u16ptr = (const U16*) ptr; - const U32 tableLog = MEM_read16(ptr); - statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; - statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); - statePtr->stateLog = tableLog; -} - - -/*! FSE_initCState2() : -* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) -* uses the smallest state value possible, saving the cost of this symbol */ -MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) -{ - FSE_initCState(statePtr, ct); - { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; - const U16* stateTable = (const U16*)(statePtr->stateTable); - U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); - statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; - statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; - } -} - -MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol) -{ - const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; - const U16* const stateTable = (const U16*)(statePtr->stateTable); - U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); - BIT_addBits(bitC, statePtr->value, nbBitsOut); - statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; -} - -MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) -{ - BIT_addBits(bitC, statePtr->value, statePtr->stateLog); - BIT_flushBits(bitC); -} - - -/* ====== Decompression ====== */ - -typedef struct { - U16 tableLog; - U16 fastMode; -} FSE_DTableHeader; /* sizeof U32 */ - -typedef struct -{ - unsigned short newState; - unsigned char symbol; - unsigned char nbBits; -} FSE_decode_t; /* size == U32 */ - -MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) -{ - const void* ptr = dt; - const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; - DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); - BIT_reloadDStream(bitD); - DStatePtr->table = dt + 1; -} - -MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; - return DInfo.symbol; -} - -MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.newState + lowBits; -} - -MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BIT_readBits(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -/*! FSE_decodeSymbolFast() : - unsafe, only works if no symbol has a probability > 50% */ -MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) -{ - FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BIT_readBitsFast(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) -{ - return DStatePtr->state == 0; -} - - - -#ifndef FSE_COMMONDEFS_ONLY - -/* ************************************************************** -* Tuning parameters -****************************************************************/ -/*!MEMORY_USAGE : -* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) -* Increasing memory usage improves compression ratio -* Reduced memory usage can improve speed, due to cache effect -* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#ifndef FSE_MAX_MEMORY_USAGE -# define FSE_MAX_MEMORY_USAGE 14 -#endif -#ifndef FSE_DEFAULT_MEMORY_USAGE -# define FSE_DEFAULT_MEMORY_USAGE 13 -#endif - -/*!FSE_MAX_SYMBOL_VALUE : -* Maximum symbol value authorized. -* Required for proper stack allocation */ -#ifndef FSE_MAX_SYMBOL_VALUE -# define FSE_MAX_SYMBOL_VALUE 255 -#endif - -/* ************************************************************** -* template functions type & suffix -****************************************************************/ -#define FSE_FUNCTION_TYPE BYTE -#define FSE_FUNCTION_EXTENSION -#define FSE_DECODE_TYPE FSE_decode_t - - -#endif /* !FSE_COMMONDEFS_ONLY */ - - -/* *************************************************************** -* Constants -*****************************************************************/ -#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) -#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX -# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" -#endif - -#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) - - -#endif /* FSE_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* FSE_H */ diff --git a/contrib/lizard/lib/entropy/fse_compress.c b/contrib/lizard/lib/entropy/fse_compress.c deleted file mode 100644 index 337b7a6ffac..00000000000 --- a/contrib/lizard/lib/entropy/fse_compress.c +++ /dev/null @@ -1,848 +0,0 @@ -/* ****************************************************************** - FSE : Finite State Entropy encoder - Copyright (C) 2013-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include /* For Visual 2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ -#else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -# else -# define FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -#endif - - -/* ************************************************************** -* Includes -****************************************************************/ -#include /* malloc, free, qsort */ -#include /* memcpy, memset */ -#include /* printf (debug) */ -#include "bitstream.h" -#define FSE_STATIC_LINKING_ONLY -#include "fse.h" - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef FSE_FUNCTION_EXTENSION -# error "FSE_FUNCTION_EXTENSION must be defined" -#endif -#ifndef FSE_FUNCTION_TYPE -# error "FSE_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define FSE_CAT(X,Y) X##Y -#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) -#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) - - -/* Function templates */ - -/* FSE_buildCTable_wksp() : - * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). - * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; - FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); - U32 const step = FSE_TABLESTEP(tableSize); - U32 cumul[FSE_MAX_SYMBOL_VALUE+2]; - - FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace; - U32 highThreshold = tableSize-1; - - /* CTable header */ - if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge); - tableU16[-2] = (U16) tableLog; - tableU16[-1] = (U16) maxSymbolValue; - - /* For explanations on how to distribute symbol values over the table : - * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ - - /* symbol start positions */ - { U32 u; - cumul[0] = 0; - for (u=1; u<=maxSymbolValue+1; u++) { - if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ - cumul[u] = cumul[u-1] + 1; - tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); - } else { - cumul[u] = cumul[u-1] + normalizedCounter[u-1]; - } } - cumul[maxSymbolValue+1] = tableSize+1; - } - - /* Spread symbols */ - { U32 position = 0; - U32 symbol; - for (symbol=0; symbol<=maxSymbolValue; symbol++) { - int nbOccurences; - for (nbOccurences=0; nbOccurences highThreshold) position = (position + step) & tableMask; /* Low proba area */ - } } - - if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ - } - - /* Build table */ - { U32 u; for (u=0; u> 3) + 3; - return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ -} - -static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, - const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, - unsigned writeIsSafe) -{ - BYTE* const ostart = (BYTE*) header; - BYTE* out = ostart; - BYTE* const oend = ostart + headerBufferSize; - int nbBits; - const int tableSize = 1 << tableLog; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - bitStream = 0; - bitCount = 0; - /* Table Size */ - bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; - bitCount += 4; - - /* Init */ - remaining = tableSize+1; /* +1 for extra accuracy */ - threshold = tableSize; - nbBits = tableLog+1; - - while (remaining>1) { /* stops at 1 */ - if (previous0) { - unsigned start = charnum; - while (!normalizedCounter[charnum]) charnum++; - while (charnum >= start+24) { - start+=24; - bitStream += 0xFFFFU << bitCount; - if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE) bitStream; - out[1] = (BYTE)(bitStream>>8); - out+=2; - bitStream>>=16; - } - while (charnum >= start+3) { - start+=3; - bitStream += 3 << bitCount; - bitCount += 2; - } - bitStream += (charnum-start) << bitCount; - bitCount += 2; - if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } } - { int count = normalizedCounter[charnum++]; - int const max = (2*threshold-1)-remaining; - remaining -= count < 0 ? -count : count; - count++; /* +1 for extra accuracy */ - if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ - bitStream += count << bitCount; - bitCount += nbBits; - bitCount -= (count>=1; - } - if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } } - - /* flush remaining bitStream */ - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out+= (bitCount+7) /8; - - if (charnum > maxSymbolValue + 1) return ERROR(GENERIC); - - return (out-ostart); -} - - -size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC); /* Unsupported */ - if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ - - if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); - - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); -} - - - -/*-************************************************************** -* Counting histogram -****************************************************************/ -/*! FSE_count_simple - This function counts byte values within `src`, and store the histogram into table `count`. - It doesn't use any additional memory. - But this function is unsafe : it doesn't check that all values within `src` can fit into `count`. - For this reason, prefer using a table `count` with 256 elements. - @return : count of most numerous element -*/ -size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, - const void* src, size_t srcSize) -{ - const BYTE* ip = (const BYTE*)src; - const BYTE* const end = ip + srcSize; - unsigned maxSymbolValue = *maxSymbolValuePtr; - unsigned max=0; - - memset(count, 0, (maxSymbolValue+1)*sizeof(*count)); - if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } - - while (ip max) max = count[s]; } - - return (size_t)max; -} - - -/* FSE_count_parallel_wksp() : - * Same as FSE_count_parallel(), but using an externally provided scratch buffer. - * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */ -static size_t FSE_count_parallel_wksp( - unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, - unsigned checkMax, unsigned* const workSpace) -{ - const BYTE* ip = (const BYTE*)source; - const BYTE* const iend = ip+sourceSize; - unsigned maxSymbolValue = *maxSymbolValuePtr; - unsigned max=0; - U32* const Counting1 = workSpace; - U32* const Counting2 = Counting1 + 256; - U32* const Counting3 = Counting2 + 256; - U32* const Counting4 = Counting3 + 256; - - memset(Counting1, 0, 4*256*sizeof(unsigned)); - - /* safety checks */ - if (!sourceSize) { - memset(count, 0, maxSymbolValue + 1); - *maxSymbolValuePtr = 0; - return 0; - } - if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */ - - /* by stripes of 16 bytes */ - { U32 cached = MEM_read32(ip); ip += 4; - while (ip < iend-15) { - U32 c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - } - ip-=4; - } - - /* finish last symbols */ - while (ipmaxSymbolValue; s--) { - Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; - if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall); - } } - - { U32 s; for (s=0; s<=maxSymbolValue; s++) { - count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; - if (count[s] > max) max = count[s]; - } } - - while (!count[maxSymbolValue]) maxSymbolValue--; - *maxSymbolValuePtr = maxSymbolValue; - return (size_t)max; -} - -/* FSE_countFast_wksp() : - * Same as FSE_countFast(), but using an externally provided scratch buffer. - * `workSpace` size must be table of >= `1024` unsigned */ -size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, unsigned* workSpace) -{ - if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); - return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace); -} - -/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ -size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize) -{ - unsigned tmpCounters[1024]; - return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters); -} - -/* FSE_count_wksp() : - * Same as FSE_count(), but using an externally provided scratch buffer. - * `workSpace` size must be table of >= `1024` unsigned */ -size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, unsigned* workSpace) -{ - if (*maxSymbolValuePtr < 255) - return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace); - *maxSymbolValuePtr = 255; - return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace); -} - -size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, - const void* src, size_t srcSize) -{ - unsigned tmpCounters[1024]; - return FSE_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters); -} - - - -/*-************************************************************** -* FSE Compression Code -****************************************************************/ -/*! FSE_sizeof_CTable() : - FSE_CTable is a variable size structure which contains : - `U16 tableLog;` - `U16 maxSymbolValue;` - `U16 nextStateNumber[1 << tableLog];` // This size is variable - `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable -Allocation is manual (C standard does not support variable-size structures). -*/ -size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog) -{ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - return FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); -} - -FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) -{ - size_t size; - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); - return (FSE_CTable*)malloc(size); -} - -void FSE_freeCTable (FSE_CTable* ct) { free(ct); } - -/* provides the minimum logSize to safely represent a distribution */ -static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) -{ - U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; - U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; - U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; - return minBits; -} - -unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) -{ - U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; - U32 tableLog = maxTableLog; - U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); - if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; - if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ - if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ - if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; - if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; - return tableLog; -} - -unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -{ - return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); -} - - -/* Secondary normalization method. - To be used when primary method fails. */ - -static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue) -{ - U32 s; - U32 distributed = 0; - U32 ToDistribute; - - /* Init */ - U32 const lowThreshold = (U32)(total >> tableLog); - U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); - - for (s=0; s<=maxSymbolValue; s++) { - if (count[s] == 0) { - norm[s]=0; - continue; - } - if (count[s] <= lowThreshold) { - norm[s] = -1; - distributed++; - total -= count[s]; - continue; - } - if (count[s] <= lowOne) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } - norm[s]=-2; - } - ToDistribute = (1 << tableLog) - distributed; - - if ((total / ToDistribute) > lowOne) { - /* risk of rounding to zero */ - lowOne = (U32)((total * 3) / (ToDistribute * 2)); - for (s=0; s<=maxSymbolValue; s++) { - if ((norm[s] == -2) && (count[s] <= lowOne)) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } } - ToDistribute = (1 << tableLog) - distributed; - } - - if (distributed == maxSymbolValue+1) { - /* all values are pretty poor; - probably incompressible data (should have already been detected); - find max, then give all remaining points to max */ - U32 maxV = 0, maxC = 0; - for (s=0; s<=maxSymbolValue; s++) - if (count[s] > maxC) maxV=s, maxC=count[s]; - norm[maxV] += (short)ToDistribute; - return 0; - } - - { U64 const vStepLog = 62 - tableLog; - U64 const mid = (1ULL << (vStepLog-1)) - 1; - U64 const rStep = ((((U64)1<> vStepLog); - U32 const sEnd = (U32)(end >> vStepLog); - U32 const weight = sEnd - sStart; - if (weight < 1) - return ERROR(GENERIC); - norm[s] = (short)weight; - tmpTotal = end; - } } } - - return 0; -} - - -size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, - const unsigned* count, size_t total, - unsigned maxSymbolValue) -{ - /* Sanity checks */ - if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; - if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ - if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ - - { U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; - U64 const scale = 62 - tableLog; - U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */ - U64 const vStep = 1ULL<<(scale-20); - int stillToDistribute = 1<> tableLog); - - for (s=0; s<=maxSymbolValue; s++) { - if (count[s] == total) return 0; /* rle special case */ - if (count[s] == 0) { normalizedCounter[s]=0; continue; } - if (count[s] <= lowThreshold) { - normalizedCounter[s] = -1; - stillToDistribute--; - } else { - short proba = (short)((count[s]*step) >> scale); - if (proba<8) { - U64 restToBeat = vStep * rtbTable[proba]; - proba += (count[s]*step) - ((U64)proba< restToBeat; - } - if (proba > largestP) largestP=proba, largest=s; - normalizedCounter[s] = proba; - stillToDistribute -= proba; - } } - if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { - /* corner case, need another normalization method */ - size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); - if (FSE_isError(errorCode)) return errorCode; - } - else normalizedCounter[largest] += (short)stillToDistribute; - } - -#if 0 - { /* Print Table (debug) */ - U32 s; - U32 nTotal = 0; - for (s=0; s<=maxSymbolValue; s++) - printf("%3i: %4i \n", s, normalizedCounter[s]); - for (s=0; s<=maxSymbolValue; s++) - nTotal += abs(normalizedCounter[s]); - if (nTotal != (1U<>1); /* assumption : tableLog >= 1 */ - FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* header */ - tableU16[-2] = (U16) nbBits; - tableU16[-1] = (U16) maxSymbolValue; - - /* Build table */ - for (s=0; s FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ - FSE_encodeSymbol(&bitC, &CState2, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - FSE_FLUSHBITS(&bitC); - } - - /* 2 or 4 encoding per loop */ - while ( ip>istart ) { - - FSE_encodeSymbol(&bitC, &CState2, *--ip); - - if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ - FSE_FLUSHBITS(&bitC); - - FSE_encodeSymbol(&bitC, &CState1, *--ip); - - if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ - FSE_encodeSymbol(&bitC, &CState2, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - } - - FSE_FLUSHBITS(&bitC); - } - - FSE_flushCState(&bitC, &CState2); - FSE_flushCState(&bitC, &CState1); - return BIT_closeCStream(&bitC); -} - -size_t FSE_compress_usingCTable (void* dst, size_t dstSize, - const void* src, size_t srcSize, - const FSE_CTable* ct) -{ - unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); - - if (fast) - return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); - else - return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); -} - - -size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } - -#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f -#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } - -/* FSE_compress_wksp() : - * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). - * `wkspSize` size must be `(1< not compressible */ - if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ - } - - tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); - CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) ); - - /* Write table description header */ - { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); - op += nc_err; - } - - /* Compress */ - CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) ); - { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); - if (cSize == 0) return 0; /* not enough space for compressed data */ - op += cSize; - } - - /* check compressibility */ - if ( (size_t)(op-ostart) >= srcSize-1 ) return 0; - - return op-ostart; -} - -typedef struct { - FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; - BYTE scratchBuffer[1 << FSE_MAX_TABLELOG]; -} fseWkspMax_t; - -size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) -{ - fseWkspMax_t scratchBuffer; - FSE_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer)); -} - -size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); -} - - -#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/contrib/lizard/lib/entropy/fse_decompress.c b/contrib/lizard/lib/entropy/fse_decompress.c deleted file mode 100644 index 1479a5e8217..00000000000 --- a/contrib/lizard/lib/entropy/fse_decompress.c +++ /dev/null @@ -1,329 +0,0 @@ -/* ****************************************************************** - FSE : Finite State Entropy decoder - Copyright (C) 2013-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include /* For Visual 2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ -#else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -# else -# define FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -#endif - - -/* ************************************************************** -* Includes -****************************************************************/ -#include /* malloc, free, qsort */ -#include /* memcpy, memset */ -#include /* printf (debug) */ -#include "bitstream.h" -#define FSE_STATIC_LINKING_ONLY -#include "fse.h" - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define FSE_isError ERR_isError -#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - -/* check and forward error code */ -#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } - - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef FSE_FUNCTION_EXTENSION -# error "FSE_FUNCTION_EXTENSION must be defined" -#endif -#ifndef FSE_FUNCTION_TYPE -# error "FSE_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define FSE_CAT(X,Y) X##Y -#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) -#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) - - -/* Function templates */ -FSE_DTable* FSE_createDTable (unsigned tableLog) -{ - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); -} - -void FSE_freeDTable (FSE_DTable* dt) -{ - free(dt); -} - -size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ - FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); - U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; - - U32 const maxSV1 = maxSymbolValue + 1; - U32 const tableSize = 1 << tableLog; - U32 highThreshold = tableSize-1; - - /* Sanity Checks */ - if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - - /* Init, lay down lowprob symbols */ - { FSE_DTableHeader DTableH; - DTableH.tableLog = (U16)tableLog; - DTableH.fastMode = 1; - { S16 const largeLimit= (S16)(1 << (tableLog-1)); - U32 s; - for (s=0; s= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; - } } } - memcpy(dt, &DTableH, sizeof(DTableH)); - } - - /* Spread symbols */ - { U32 const tableMask = tableSize-1; - U32 const step = FSE_TABLESTEP(tableSize); - U32 s, position = 0; - for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ - } } - if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ - } - - /* Build Decoding table */ - { U32 u; - for (u=0; utableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - - -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) -{ - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask+1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s=0; s sizeof(bitD.bitContainer)*8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[1] = FSE_GETSYMBOL(&state2); - - if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } - - op[2] = FSE_GETSYMBOL(&state1); - - if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[3] = FSE_GETSYMBOL(&state2); - } - - /* tail */ - /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ - while (1) { - if (op>(omax-2)) return ERROR(dstSize_tooSmall); - *op++ = FSE_GETSYMBOL(&state1); - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { - *op++ = FSE_GETSYMBOL(&state2); - break; - } - - if (op>(omax-2)) return ERROR(dstSize_tooSmall); - *op++ = FSE_GETSYMBOL(&state2); - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { - *op++ = FSE_GETSYMBOL(&state1); - break; - } } - - return op-ostart; -} - - -size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize, - const FSE_DTable* dt) -{ - const void* ptr = dt; - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; - const U32 fastMode = DTableH->fastMode; - - /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -} - - -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog) -{ - const BYTE* const istart = (const BYTE*)cSrc; - const BYTE* ip = istart; - short counting[FSE_MAX_SYMBOL_VALUE+1]; - unsigned tableLog; - unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; - - /* normal FSE decoding mode */ - size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); - if (FSE_isError(NCountLength)) return NCountLength; - //if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */ - if (tableLog > maxLog) return ERROR(tableLog_tooLarge); - ip += NCountLength; - cSrcSize -= NCountLength; - - CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) ); - - return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */ -} - - -typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; - -size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) -{ - DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ - return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG); -} - - - -#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/contrib/lizard/lib/entropy/huf.h b/contrib/lizard/lib/entropy/huf.h deleted file mode 100644 index 48c7f831f55..00000000000 --- a/contrib/lizard/lib/entropy/huf.h +++ /dev/null @@ -1,250 +0,0 @@ -/* ****************************************************************** - Huffman coder, part of New Generation Entropy library - header file - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef HUF_H_298734234 -#define HUF_H_298734234 - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* *** Dependencies *** */ -#include /* size_t */ - - -/*-*** PUBLIC_API : control library symbols visibility *** */ -#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) -# define HUF_PUBLIC_API __attribute__ ((visibility ("default"))) -#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ -# define HUF_PUBLIC_API __declspec(dllexport) -#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) -# define HUF_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define HUF_PUBLIC_API -#endif - - -/* *** simple functions *** */ -/** -HUF_compress() : - Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. - 'dst' buffer must be already allocated. - Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). - `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. - @return : size of compressed data (<= `dstCapacity`). - Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - if return == 1, srcData is a single repeated byte symbol (RLE compression). - if HUF_isError(return), compression failed (more details using HUF_getErrorName()) -*/ -HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/** -HUF_decompress() : - Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', - into already allocated buffer 'dst', of minimum size 'dstSize'. - `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. - Note : in contrast with FSE, HUF_decompress can regenerate - RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, - because it knows size to regenerate. - @return : size of regenerated data (== originalSize), - or an error code, which can be tested using HUF_isError() -*/ -HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize); - - -/* *** Tool functions *** */ -#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ -HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ - -/* Error Management */ -HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ -HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ - - -/* *** Advanced function *** */ - -/** HUF_compress2() : - * Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` . - * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ -HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); - -/** HUF_compress4X_wksp() : -* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */ -HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least 1024 unsigned */ - - - -#ifdef HUF_STATIC_LINKING_ONLY - -/* *** Dependencies *** */ -#include "mem.h" /* U32 */ - - -/* *** Constants *** */ -#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ -#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ -#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ -#define HUF_SYMBOLVALUE_MAX 255 -#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) -# error "HUF_TABLELOG_MAX is too large !" -#endif - - -/* **************************************** -* Static allocation -******************************************/ -/* HUF buffer bounds */ -#define HUF_CTABLEBOUND 129 -#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ -#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ - -/* static allocation of HUF's Compression Table */ -#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ - U32 name##hb[maxSymbolValue+1]; \ - void* name##hv = &(name##hb); \ - HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ - -/* static allocation of HUF's DTable */ -typedef U32 HUF_DTable; -#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) -#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ - HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } -#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ - HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } - - -/* **************************************** -* Advanced decompression functions -******************************************/ -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ - -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ -size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ - - -/* **************************************** -* HUF detailed API -******************************************/ -/*! -HUF_compress() does the following: -1. count symbol occurrence from source[] into table count[] using FSE_count() -2. (optional) refine tableLog using HUF_optimalTableLog() -3. build Huffman table from count using HUF_buildCTable() -4. save Huffman table to memory buffer using HUF_writeCTable() -5. encode the data stream using HUF_compress4X_usingCTable() - -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and regenerate 'CTable' using external methods. -*/ -/* FSE_count() : find it within "fse.h" */ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ -size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); -size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); - - -/** HUF_buildCTable_wksp() : - * Same as HUF_buildCTable(), but using externally allocated scratch buffer. - * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. - */ -size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize); - -/*! HUF_readStats() : - Read compact Huffman tree, saved by HUF_writeCTable(). - `huffWeight` is destination buffer. - @return : size read from `src` , or an error Code . - Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ -size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, - U32* nbSymbolsPtr, U32* tableLogPtr, - const void* src, size_t srcSize); - -/** HUF_readCTable() : -* Loading a CTable saved with HUF_writeCTable() */ -size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); - - -/* -HUF_decompress() does the following: -1. select the decompression algorithm (X2, X4) based on pre-computed heuristics -2. build Huffman table from save, using HUF_readDTableXn() -3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable -*/ - -/** HUF_selectDecoder() : -* Tells which decoder is likely to decode faster, -* based on a set of pre-determined metrics. -* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . -* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); - -size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize); - -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); - - -/* single stream variants */ - -size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least 1024 unsigned */ -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); - -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ -size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ - -size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); -size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ - -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ -size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); - -#endif /* HUF_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* HUF_H_298734234 */ diff --git a/contrib/lizard/lib/entropy/huf_compress.c b/contrib/lizard/lib/entropy/huf_compress.c deleted file mode 100644 index a47a1a2c761..00000000000 --- a/contrib/lizard/lib/entropy/huf_compress.c +++ /dev/null @@ -1,612 +0,0 @@ -/* ****************************************************************** - Huffman encoder, part of New Generation Entropy library - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - - -/* ************************************************************** -* Includes -****************************************************************/ -#include /* memcpy, memset */ -#include /* printf (debug) */ -#include "bitstream.h" -#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ -#include "fse.h" /* header compression */ -#define HUF_STATIC_LINKING_ONLY -#include "huf.h" - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ -#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f -#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } - - -/* ************************************************************** -* Utils -****************************************************************/ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -{ - return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); -} - - -/* ******************************************************* -* HUF : Huffman block compression -*********************************************************/ -/* HUF_compressWeights() : - * Same as FSE_compress(), but dedicated to huff0's weights compression. - * The use case needs much less stack memory. - * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. - */ -#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 -size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize) -{ - BYTE* const ostart = (BYTE*) dst; - BYTE* op = ostart; - BYTE* const oend = ostart + dstSize; - - U32 maxSymbolValue = HUF_TABLELOG_MAX; - U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; - - FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; - BYTE scratchBuffer[1< not compressible */ - } - - tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); - CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) ); - - /* Write table description header */ - { CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); - op += hSize; - } - - /* Compress */ - CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) ); - { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) ); - if (cSize == 0) return 0; /* not enough space for compressed data */ - op += cSize; - } - - return op-ostart; -} - - -struct HUF_CElt_s { - U16 val; - BYTE nbBits; -}; /* typedef'd to HUF_CElt within "huf.h" */ - -/*! HUF_writeCTable() : - `CTable` : huffman tree to save, using huf representation. - @return : size of saved CTable */ -size_t HUF_writeCTable (void* dst, size_t maxDstSize, - const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog) -{ - BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ - BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; - BYTE* op = (BYTE*)dst; - U32 n; - - /* check conditions */ - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); - - /* convert to weight */ - bitsToWeight[0] = 0; - for (n=1; n1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ - op[0] = (BYTE)hSize; - return hSize+1; - } } - - /* write raw values as 4-bits (max : 15) */ - if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ - if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ - op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); - huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ - for (n=0; n HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall); - - /* Prepare base value per rank */ - { U32 n, nextRankStart = 0; - for (n=1; n<=tableLog; n++) { - U32 current = nextRankStart; - nextRankStart += (rankVal[n] << (n-1)); - rankVal[n] = current; - } } - - /* fill nbBits */ - { U32 n; for (n=0; nn=tableLog+1 */ - U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; - { U32 n; for (n=0; n0; n--) { /* start at n=tablelog <-> w=1 */ - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - /* assign value within rank, symbol order */ - { U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; } - } - - return readSize; -} - - -typedef struct nodeElt_s { - U32 count; - U16 parent; - BYTE byte; - BYTE nbBits; -} nodeElt; - -static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) -{ - const U32 largestBits = huffNode[lastNonNull].nbBits; - if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */ - - /* there are several too large elements (at least >= 2) */ - { int totalCost = 0; - const U32 baseCost = 1 << (largestBits - maxNbBits); - U32 n = lastNonNull; - - while (huffNode[n].nbBits > maxNbBits) { - totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); - huffNode[n].nbBits = (BYTE)maxNbBits; - n --; - } /* n stops at huffNode[n].nbBits <= maxNbBits */ - while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ - - /* renorm totalCost */ - totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ - - /* repay normalized cost */ - { U32 const noSymbol = 0xF0F0F0F0; - U32 rankLast[HUF_TABLELOG_MAX+2]; - int pos; - - /* Get pos of last (smallest) symbol per rank */ - memset(rankLast, 0xF0, sizeof(rankLast)); - { U32 currentNbBits = maxNbBits; - for (pos=n ; pos >= 0; pos--) { - if (huffNode[pos].nbBits >= currentNbBits) continue; - currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ - rankLast[maxNbBits-currentNbBits] = pos; - } } - - while (totalCost > 0) { - U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; - for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { - U32 highPos = rankLast[nBitsToDecrease]; - U32 lowPos = rankLast[nBitsToDecrease-1]; - if (highPos == noSymbol) continue; - if (lowPos == noSymbol) break; - { U32 const highTotal = huffNode[highPos].count; - U32 const lowTotal = 2 * huffNode[lowPos].count; - if (highTotal <= lowTotal) break; - } } - /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ - while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ - nBitsToDecrease ++; - totalCost -= 1 << (nBitsToDecrease-1); - if (rankLast[nBitsToDecrease-1] == noSymbol) - rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ - huffNode[rankLast[nBitsToDecrease]].nbBits ++; - if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ - rankLast[nBitsToDecrease] = noSymbol; - else { - rankLast[nBitsToDecrease]--; - if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) - rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ - } } /* while (totalCost > 0) */ - - while (totalCost < 0) { /* Sometimes, cost correction overshoot */ - if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ - while (huffNode[n].nbBits == maxNbBits) n--; - huffNode[n+1].nbBits--; - rankLast[1] = n+1; - totalCost++; - continue; - } - huffNode[ rankLast[1] + 1 ].nbBits--; - rankLast[1]++; - totalCost ++; - } } } /* there are several too large elements (at least >= 2) */ - - return maxNbBits; -} - - -typedef struct { - U32 base; - U32 current; -} rankPos; - -static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue) -{ - rankPos rank[32]; - U32 n; - - memset(rank, 0, sizeof(rank)); - for (n=0; n<=maxSymbolValue; n++) { - U32 r = BIT_highbit32(count[n] + 1); - rank[r].base ++; - } - for (n=30; n>0; n--) rank[n-1].base += rank[n].base; - for (n=0; n<32; n++) rank[n].current = rank[n].base; - for (n=0; n<=maxSymbolValue; n++) { - U32 const c = count[n]; - U32 const r = BIT_highbit32(c+1) + 1; - U32 pos = rank[r].current++; - while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--; - huffNode[pos].count = c; - huffNode[pos].byte = (BYTE)n; - } -} - - -/** HUF_buildCTable_wksp() : - * Same as HUF_buildCTable(), but using externally allocated scratch buffer. - * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. - */ -#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) -typedef nodeElt huffNodeTable[2*HUF_SYMBOLVALUE_MAX+1 +1]; -size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize) -{ - nodeElt* const huffNode0 = (nodeElt*)workSpace; - nodeElt* const huffNode = huffNode0+1; - U32 n, nonNullRank; - int lowS, lowN; - U16 nodeNb = STARTNODE; - U32 nodeRoot; - - /* safety checks */ - if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); /* workSpace is not large enough */ - if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC); - memset(huffNode0, 0, sizeof(huffNodeTable)); - - /* sort, decreasing order */ - HUF_sort(huffNode, count, maxSymbolValue); - - /* init for parents */ - nonNullRank = maxSymbolValue; - while(huffNode[nonNullRank].count == 0) nonNullRank--; - lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; - huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; - huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb; - nodeNb++; lowS-=2; - for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); - huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ - - /* create parents */ - while (nodeNb <= nodeRoot) { - U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; - huffNode[n1].parent = huffNode[n2].parent = nodeNb; - nodeNb++; - } - - /* distribute weights (unlimited tree height) */ - huffNode[nodeRoot].nbBits = 0; - for (n=nodeRoot-1; n>=STARTNODE; n--) - huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; - for (n=0; n<=nonNullRank; n++) - huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; - - /* enforce maxTableLog */ - maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); - - /* fill result into tree (val, nbBits) */ - { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; - U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; - if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ - for (n=0; n<=nonNullRank; n++) - nbPerRank[huffNode[n].nbBits]++; - /* determine stating value per rank */ - { U16 min = 0; - for (n=maxNbBits; n>0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - for (n=0; n<=maxSymbolValue; n++) - tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ - for (n=0; n<=maxSymbolValue; n++) - tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ - } - - return maxNbBits; -} - -/** HUF_buildCTable() : - * Note : count is used before tree is written, so they can safely overlap - */ -size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) -{ - huffNodeTable nodeTable; - return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable)); -} - -static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) -{ - BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); -} - -size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } - -#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) - -#define HUF_FLUSHBITS_1(stream) \ - if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream) - -#define HUF_FLUSHBITS_2(stream) \ - if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream) - -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) -{ - const BYTE* ip = (const BYTE*) src; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - size_t n; - const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize)); - BIT_CStream_t bitC; - - /* init */ - if (dstSize < 8) return 0; /* not enough space to compress */ - { size_t const initErr = BIT_initCStream(&bitC, op, oend-op); - if (HUF_isError(initErr)) return 0; } - - n = srcSize & ~3; /* join to mod 4 */ - switch (srcSize & 3) - { - case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); - HUF_FLUSHBITS_2(&bitC); - // fallthrough - case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); - HUF_FLUSHBITS_1(&bitC); - // fallthrough - case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable); - HUF_FLUSHBITS(&bitC); - // fallthrough - case 0 : - default: ; - } - - for (; n>0; n-=4) { /* note : n&3==0 at this stage */ - HUF_encodeSymbol(&bitC, ip[n- 1], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 2], CTable); - HUF_FLUSHBITS_2(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 3], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 4], CTable); - HUF_FLUSHBITS(&bitC); - } - - return BIT_closeCStream(&bitC); -} - - -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) -{ - size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ - const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - - if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ - if (srcSize < 12) return 0; /* no saving possible : too small input */ - op += 6; /* jumpTable */ - - { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); - if (cSize==0) return 0; - MEM_writeLE16(ostart, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); - if (cSize==0) return 0; - MEM_writeLE16(ostart+2, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); - if (cSize==0) return 0; - MEM_writeLE16(ostart+4, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) ); - if (cSize==0) return 0; - op += cSize; - } - - return op-ostart; -} - - -/* `workSpace` must a table of at least 1024 unsigned */ -static size_t HUF_compress_internal ( - void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - unsigned singleStream, - void* workSpace, size_t wkspSize) -{ - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - - union { - U32 count[HUF_SYMBOLVALUE_MAX+1]; - HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1]; - } table; /* `count` can overlap with `CTable`; saves 1 KB */ - - /* checks & inits */ - if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); - if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */ - if (!dstSize) return 0; /* cannot fit within dst budget */ - if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ - if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; - if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; - - /* Scan input and build symbol stats */ - { CHECK_V_F(largest, FSE_count_wksp (table.count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) ); - if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ - if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ - } - - /* Build Huffman Tree */ - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); - { CHECK_V_F(maxBits, HUF_buildCTable_wksp (table.CTable, table.count, maxSymbolValue, huffLog, workSpace, wkspSize) ); - huffLog = (U32)maxBits; - } - - /* Write table description header */ - { CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, table.CTable, maxSymbolValue, huffLog) ); - if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */ - op += hSize; - } - - /* Compress */ - { size_t const cSize = (singleStream) ? - HUF_compress1X_usingCTable(op, oend - op, src, srcSize, table.CTable) : /* single segment */ - HUF_compress4X_usingCTable(op, oend - op, src, srcSize, table.CTable); - if (HUF_isError(cSize)) return cSize; - if (cSize==0) return 0; /* uncompressible */ - op += cSize; - } - - /* check compressibility */ - if ((size_t)(op-ostart) >= srcSize-1) - return 0; - - return op-ostart; -} - - -size_t HUF_compress1X_wksp (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - void* workSpace, size_t wkspSize) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize); -} - -size_t HUF_compress1X (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - unsigned workSpace[1024]; - return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); -} - -size_t HUF_compress4X_wksp (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - void* workSpace, size_t wkspSize) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize); -} - -size_t HUF_compress2 (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - unsigned workSpace[1024]; - return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); -} - -size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) -{ - return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT); -} diff --git a/contrib/lizard/lib/entropy/huf_decompress.c b/contrib/lizard/lib/entropy/huf_decompress.c deleted file mode 100644 index a342dfb1e4a..00000000000 --- a/contrib/lizard/lib/entropy/huf_decompress.c +++ /dev/null @@ -1,885 +0,0 @@ -/* ****************************************************************** - Huffman decoder, part of New Generation Entropy library - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -/* inline is defined */ -#elif defined(_MSC_VER) || defined(__GNUC__) -# define inline __inline -#else -# define inline /* disable inline */ -#endif - -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - - -/* ************************************************************** -* Dependencies -****************************************************************/ -#include /* memcpy, memset */ -#include "bitstream.h" /* BIT_* */ -#include "fse.h" /* header compression */ -#define HUF_STATIC_LINKING_ONLY -#include "huf.h" - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/*-***************************/ -/* generic DTableDesc */ -/*-***************************/ - -typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; - -static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) -{ - DTableDesc dtd; - memcpy(&dtd, table, sizeof(dtd)); - return dtd; -} - - -/*-***************************/ -/* single-symbol decoding */ -/*-***************************/ - -typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ - -size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize) -{ - BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; - U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ - U32 tableLog = 0; - U32 nbSymbols = 0; - size_t iSize; - void* const dtPtr = DTable + 1; - HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; - - HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); - /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ - - iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); - if (HUF_isError(iSize)) return iSize; - - /* Table header */ - { DTableDesc dtd = HUF_getDTableDesc(DTable); - if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */ - dtd.tableType = 0; - dtd.tableLog = (BYTE)tableLog; - memcpy(DTable, &dtd, sizeof(dtd)); - } - - /* Prepare ranks */ - { U32 n, nextRankStart = 0; - for (n=1; n> 1; - U32 i; - HUF_DEltX2 D; - D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); - for (i = rankVal[w]; i < rankVal[w] + length; i++) - dt[i] = D; - rankVal[w] += length; - } } - - return iSize; -} - - -static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ - BYTE const c = dt[val].byte; - BIT_skipBits(Dstream, dt[val].nbBits); - return c; -} - -#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ - *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ - HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; - - /* up to 4 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) { - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_1(p, bitDPtr); - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - } - - /* closer to the end */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - - /* no more data to retrieve from bitstream, hence no need to reload */ - while (p < pEnd) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - - return pEnd-pStart; -} - -static size_t HUF_decompress1X2_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + dstSize; - const void* dtPtr = DTable + 1; - const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; - BIT_DStream_t bitD; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); - if (HUF_isError(errorCode)) return errorCode; } - - HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); - - /* check */ - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - - return dstSize; -} - -size_t HUF_decompress1X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX2 (DCtx, cSrc, cSrcSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); -} - -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); -} - - -static size_t HUF_decompress4X2_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - /* Check */ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable + 1; - const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - U32 endSignal; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUF_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUF_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUF_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUF_isError(errorCode)) return errorCode; } - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_1(op1, &bitD1); - HUF_DECODE_SYMBOLX2_1(op2, &bitD2); - HUF_DECODE_SYMBOLX2_1(op3, &bitD3); - HUF_DECODE_SYMBOLX2_1(op4, &bitD4); - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_0(op1, &bitD1); - HUF_DECODE_SYMBOLX2_0(op2, &bitD2); - HUF_DECODE_SYMBOLX2_0(op3, &bitD3); - HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ - - /* finish bitStreams one by one */ - HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); - - /* check */ - endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endSignal) return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; - } -} - - -size_t HUF_decompress4X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - - -size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX2 (dctx, cSrc, cSrcSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx); -} - -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - - -/* *************************/ -/* double-symbols decoding */ -/* *************************/ -typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ - -typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; - -/* HUF_fillDTableX4Level2() : - * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ -static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, - const U32* rankValOrigin, const int minWeight, - const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, - U32 nbBitsBaseline, U16 baseSeq) -{ - HUF_DEltX4 DElt; - U32 rankVal[HUF_TABLELOG_MAX + 1]; - - /* get pre-calculated rankVal */ - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill skipped values */ - if (minWeight>1) { - U32 i, skipSize = rankVal[minWeight]; - MEM_writeLE16(&(DElt.sequence), baseSeq); - DElt.nbBits = (BYTE)(consumed); - DElt.length = 1; - for (i = 0; i < skipSize; i++) - DTable[i] = DElt; - } - - /* fill DTable */ - { U32 s; for (s=0; s= 1 */ - - rankVal[weight] += length; - } } -} - -typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1]; - -static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, - const sortedSymbol_t* sortedList, const U32 sortedListSize, - const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, - const U32 nbBitsBaseline) -{ - U32 rankVal[HUF_TABLELOG_MAX + 1]; - const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ - const U32 minBits = nbBitsBaseline - maxWeight; - U32 s; - - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill DTable */ - for (s=0; s= minBits) { /* enough room for a second symbol */ - U32 sortedRank; - int minWeight = nbBits + scaleLog; - if (minWeight < 1) minWeight = 1; - sortedRank = rankStart[minWeight]; - HUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, - rankValOrigin[nbBits], minWeight, - sortedList+sortedRank, sortedListSize-sortedRank, - nbBitsBaseline, symbol); - } else { - HUF_DEltX4 DElt; - MEM_writeLE16(&(DElt.sequence), symbol); - DElt.nbBits = (BYTE)(nbBits); - DElt.length = 1; - { U32 const end = start + length; - U32 u; - for (u = start; u < end; u++) DTable[u] = DElt; - } } - rankVal[weight] += length; - } -} - -size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) -{ - BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; - sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; - U32 rankStats[HUF_TABLELOG_MAX + 1] = { 0 }; - U32 rankStart0[HUF_TABLELOG_MAX + 2] = { 0 }; - U32* const rankStart = rankStart0+1; - rankVal_t rankVal; - U32 tableLog, maxW, sizeOfSort, nbSymbols; - DTableDesc dtd = HUF_getDTableDesc(DTable); - U32 const maxTableLog = dtd.maxTableLog; - size_t iSize; - void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ - HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr; - - HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */ - if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ - - iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); - if (HUF_isError(iSize)) return iSize; - - /* check result */ - if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ - - /* find maxWeight */ - for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ - - /* Get start index of each weight */ - { U32 w, nextRankStart = 0; - for (w=1; w> consumed; - } } } } - - HUF_fillDTableX4(dt, maxTableLog, - sortedSymbol, sizeOfSort, - rankStart0, rankVal, maxW, - tableLog+1); - - dtd.tableLog = (BYTE)maxTableLog; - dtd.tableType = 1; - memcpy(DTable, &dtd, sizeof(dtd)); - return iSize; -} - - -static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt+val, 2); - BIT_skipBits(DStream, dt[val].nbBits); - return dt[val].length; -} - -static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt+val, 1); - if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); - else { - if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { - BIT_skipBits(DStream, dt[val].nbBits); - if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) - DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ - } } - return 1; -} - - -#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ - ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ - ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; - - /* up to 8 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { - HUF_DECODE_SYMBOLX4_2(p, bitDPtr); - HUF_DECODE_SYMBOLX4_1(p, bitDPtr); - HUF_DECODE_SYMBOLX4_2(p, bitDPtr); - HUF_DECODE_SYMBOLX4_0(p, bitDPtr); - } - - /* closer to end : up to 2 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) - HUF_DECODE_SYMBOLX4_0(p, bitDPtr); - - while (p <= pEnd-2) - HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ - - if (p < pEnd) - p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); - - return p-pStart; -} - - -static size_t HUF_decompress1X4_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - BIT_DStream_t bitD; - - /* Init */ - { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); - if (HUF_isError(errorCode)) return errorCode; - } - - /* decode */ - { BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ - const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); - } - - /* check */ - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; -} - -size_t HUF_decompress1X4_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX4 (DCtx, cSrc, cSrcSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); -} - -size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - -static size_t HUF_decompress4X4_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable+1; - const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - size_t const segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - U32 endSignal; - DTableDesc const dtd = HUF_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUF_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUF_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUF_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUF_isError(errorCode)) return errorCode; } - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) { - HUF_DECODE_SYMBOLX4_2(op1, &bitD1); - HUF_DECODE_SYMBOLX4_2(op2, &bitD2); - HUF_DECODE_SYMBOLX4_2(op3, &bitD3); - HUF_DECODE_SYMBOLX4_2(op4, &bitD4); - HUF_DECODE_SYMBOLX4_1(op1, &bitD1); - HUF_DECODE_SYMBOLX4_1(op2, &bitD2); - HUF_DECODE_SYMBOLX4_1(op3, &bitD3); - HUF_DECODE_SYMBOLX4_1(op4, &bitD4); - HUF_DECODE_SYMBOLX4_2(op1, &bitD1); - HUF_DECODE_SYMBOLX4_2(op2, &bitD2); - HUF_DECODE_SYMBOLX4_2(op3, &bitD3); - HUF_DECODE_SYMBOLX4_2(op4, &bitD4); - HUF_DECODE_SYMBOLX4_0(op1, &bitD1); - HUF_DECODE_SYMBOLX4_0(op2, &bitD2); - HUF_DECODE_SYMBOLX4_0(op3, &bitD3); - HUF_DECODE_SYMBOLX4_0(op4, &bitD4); - - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 already verified within main loop */ - - /* finish bitStreams one by one */ - HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); - - /* check */ - { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endCheck) return ERROR(corruption_detected); } - - /* decoded size */ - return dstSize; - } -} - - -size_t HUF_decompress4X4_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - - -size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t hSize = HUF_readDTableX4 (dctx, cSrc, cSrcSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); -} - -size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - - -/* ********************************/ -/* Generic decompression selector */ -/* ********************************/ - -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); - return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : - HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -} - -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); - return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : - HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -} - - -typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; -static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = -{ - /* single, double, quad */ - {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ - {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ - {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ - {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ - {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ - {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ - {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ - {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ - {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ - {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ - {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ - {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ - {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ - {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ - {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ - {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ -}; - -/** HUF_selectDecoder() : -* Tells which decoder is likely to decode faster, -* based on a set of pre-determined metrics. -* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . -* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) -{ - /* decoder timing evaluation */ - U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ - U32 const D256 = (U32)(dstSize >> 8); - U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); - U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); - DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ - - return DTime1 < DTime0; -} - - -typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); - -size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - static const decompressionAlgo decompress[2] = { HUF_decompress4X2, HUF_decompress4X4 }; - - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); - return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); - } -} - -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; - } -} - -size_t HUF_decompress4X_hufOnly (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; - } -} - -size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; - } -} diff --git a/contrib/lizard/lib/entropy/mem.h b/contrib/lizard/lib/entropy/mem.h deleted file mode 100644 index 708d897a1cc..00000000000 --- a/contrib/lizard/lib/entropy/mem.h +++ /dev/null @@ -1,372 +0,0 @@ -/** - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ - -#ifndef MEM_H_MODULE -#define MEM_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - -/*-**************************************** -* Dependencies -******************************************/ -#include /* size_t, ptrdiff_t */ -#include /* memcpy */ - - -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(_MSC_VER) /* Visual Studio */ -# include /* _byteswap_ulong */ -# include /* _byteswap_* */ -#endif -#if defined(__GNUC__) -# define MEM_STATIC static __inline __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define MEM_STATIC static inline -#elif defined(_MSC_VER) -# define MEM_STATIC static __inline -#else -# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - -/* code only tested on 32 and 64 bits systems */ -#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } -MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } - - -/*-************************************************************** -* Basic Types -*****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; - typedef intptr_t iPtrDiff; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; - typedef ptrdiff_t iPtrDiff; -#endif - - -/*-************************************************************** -* Memory I/O -*****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) -# define MEM_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } -MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } - -MEM_STATIC unsigned MEM_isLittleEndian(void) -{ - const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ - return one.c[0]; -} - -#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) - -/* violates C standard, by lying on structure alignment. -Only use if no other choice to achieve best performance on target platform */ -MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } -MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } -MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } -MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } - -#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) - __pragma( pack(push, 1) ) - typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign; - __pragma( pack(pop) ) -#else - typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; -#endif - -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } -MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } - -#else - -/* default method, safe and standard. - can sometimes prove slower */ - -MEM_STATIC U16 MEM_read16(const void* memPtr) -{ - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC U32 MEM_read32(const void* memPtr) -{ - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC U64 MEM_read64(const void* memPtr) -{ - U64 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC size_t MEM_readST(const void* memPtr) -{ - size_t val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -MEM_STATIC void MEM_write32(void* memPtr, U32 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -MEM_STATIC void MEM_write64(void* memPtr, U64 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -#endif /* MEM_FORCE_MEMORY_ACCESS */ - -MEM_STATIC U32 MEM_swap32(U32 in) -{ -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_ulong(in); -#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403) - return __builtin_bswap32(in); -#else - return ((in << 24) & 0xff000000 ) | - ((in << 8) & 0x00ff0000 ) | - ((in >> 8) & 0x0000ff00 ) | - ((in >> 24) & 0x000000ff ); -#endif -} - -MEM_STATIC U64 MEM_swap64(U64 in) -{ -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_uint64(in); -#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403) - return __builtin_bswap64(in); -#else - return ((in << 56) & 0xff00000000000000ULL) | - ((in << 40) & 0x00ff000000000000ULL) | - ((in << 24) & 0x0000ff0000000000ULL) | - ((in << 8) & 0x000000ff00000000ULL) | - ((in >> 8) & 0x00000000ff000000ULL) | - ((in >> 24) & 0x0000000000ff0000ULL) | - ((in >> 40) & 0x000000000000ff00ULL) | - ((in >> 56) & 0x00000000000000ffULL); -#endif -} - -MEM_STATIC size_t MEM_swapST(size_t in) -{ - if (MEM_32bits()) - return (size_t)MEM_swap32((U32)in); - else - return (size_t)MEM_swap64((U64)in); -} - -/*=== Little endian r/w ===*/ - -MEM_STATIC U16 MEM_readLE16(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_read16(memPtr); - else { - const BYTE* p = (const BYTE*)memPtr; - return (U16)(p[0] + (p[1]<<8)); - } -} - -MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) -{ - if (MEM_isLittleEndian()) { - MEM_write16(memPtr, val); - } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE)val; - p[1] = (BYTE)(val>>8); - } -} - -MEM_STATIC U32 MEM_readLE24(const void* memPtr) -{ - return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); -} - -MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) -{ - MEM_writeLE16(memPtr, (U16)val); - ((BYTE*)memPtr)[2] = (BYTE)(val>>16); -} - -MEM_STATIC U32 MEM_readLE32(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_read32(memPtr); - else - return MEM_swap32(MEM_read32(memPtr)); -} - -MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) -{ - if (MEM_isLittleEndian()) - MEM_write32(memPtr, val32); - else - MEM_write32(memPtr, MEM_swap32(val32)); -} - -MEM_STATIC U64 MEM_readLE64(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_read64(memPtr); - else - return MEM_swap64(MEM_read64(memPtr)); -} - -MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) -{ - if (MEM_isLittleEndian()) - MEM_write64(memPtr, val64); - else - MEM_write64(memPtr, MEM_swap64(val64)); -} - -MEM_STATIC size_t MEM_readLEST(const void* memPtr) -{ - if (MEM_32bits()) - return (size_t)MEM_readLE32(memPtr); - else - return (size_t)MEM_readLE64(memPtr); -} - -MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) -{ - if (MEM_32bits()) - MEM_writeLE32(memPtr, (U32)val); - else - MEM_writeLE64(memPtr, (U64)val); -} - -/*=== Big endian r/w ===*/ - -MEM_STATIC U32 MEM_readBE32(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_swap32(MEM_read32(memPtr)); - else - return MEM_read32(memPtr); -} - -MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) -{ - if (MEM_isLittleEndian()) - MEM_write32(memPtr, MEM_swap32(val32)); - else - MEM_write32(memPtr, val32); -} - -MEM_STATIC U64 MEM_readBE64(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_swap64(MEM_read64(memPtr)); - else - return MEM_read64(memPtr); -} - -MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) -{ - if (MEM_isLittleEndian()) - MEM_write64(memPtr, MEM_swap64(val64)); - else - MEM_write64(memPtr, val64); -} - -MEM_STATIC size_t MEM_readBEST(const void* memPtr) -{ - if (MEM_32bits()) - return (size_t)MEM_readBE32(memPtr); - else - return (size_t)MEM_readBE64(memPtr); -} - -MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) -{ - if (MEM_32bits()) - MEM_writeBE32(memPtr, (U32)val); - else - MEM_writeBE64(memPtr, (U64)val); -} - - -/* function safe only for comparisons */ -MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) -{ - switch (length) - { - default : - case 4 : return MEM_read32(memPtr); - case 3 : if (MEM_isLittleEndian()) - return MEM_read32(memPtr)<<8; - else - return MEM_read32(memPtr)>>8; - } -} - -#if defined (__cplusplus) -} -#endif - -#endif /* MEM_H_MODULE */ diff --git a/contrib/lizard/lib/liblizard.pc.in b/contrib/lizard/lib/liblizard.pc.in deleted file mode 100644 index 233c2e08faf..00000000000 --- a/contrib/lizard/lib/liblizard.pc.in +++ /dev/null @@ -1,15 +0,0 @@ -# Lizard - Fast LZ compression algorithm -# Copyright (C) 2011-2014, Yann Collet -# Copyright (C) 2016-2017, Przemyslaw Skibinski -# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -prefix=@PREFIX@ -libdir=@LIBDIR@ -includedir=@INCLUDEDIR@ - -Name: lizard -Description: extremely fast lossless compression algorithm library -URL: http://github.com/inikep/lizard -Version: @VERSION@ -Libs: -L@LIBDIR@ -llizard -Cflags: -I@INCLUDEDIR@ diff --git a/contrib/lizard/lib/lizard_common.h b/contrib/lizard/lib/lizard_common.h deleted file mode 100644 index 45730cf4e86..00000000000 --- a/contrib/lizard/lib/lizard_common.h +++ /dev/null @@ -1,504 +0,0 @@ -/* - Lizard - Fast LZ compression algorithm - Copyright (C) 2011-2015, Yann Collet - Copyright (C) 2016-2017, Przemyslaw Skibinski - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ - -#ifndef LIZARD_COMMON_H_2983 -#define LIZARD_COMMON_H_2983 - -#if defined (__cplusplus) -extern "C" { -#endif - - -/*-************************************ -* Memory routines -**************************************/ -#include /* malloc, calloc, free */ -#include /* memset, memcpy */ -#include /* intptr_t */ -#include "entropy/mem.h" -#include "lizard_compress.h" /* LIZARD_GCC_VERSION */ - -//#define LIZARD_USE_LOGS -#define LIZARD_LOG_COMPRESS(...) //printf(__VA_ARGS__) -#define LIZARD_LOG_DECOMPRESS(...) //printf(__VA_ARGS__) - -#define LIZARD_LOG_COMPRESS_LZ4(...) //printf(__VA_ARGS__) -#define COMPLOG_CODEWORDS_LZ4(...) //printf(__VA_ARGS__) -#define LIZARD_LOG_DECOMPRESS_LZ4(...) //printf(__VA_ARGS__) -#define DECOMPLOG_CODEWORDS_LZ4(...) //printf(__VA_ARGS__) - -#define LIZARD_LOG_COMPRESS_LIZv1(...) //printf(__VA_ARGS__) -#define COMPLOG_CODEWORDS_LIZv1(...) //printf(__VA_ARGS__) -#define LIZARD_LOG_DECOMPRESS_LIZv1(...) //printf(__VA_ARGS__) -#define DECOMPLOG_CODEWORDS_LIZv1(...) //printf(__VA_ARGS__) - - - - -/*-************************************ -* Common Constants -**************************************/ -#define MINMATCH 4 -//#define USE_LZ4_ONLY -//#define LIZARD_USE_TEST - -#define LIZARD_DICT_SIZE (1<<24) -#define WILDCOPYLENGTH 16 -#define LASTLITERALS WILDCOPYLENGTH -#define MFLIMIT (WILDCOPYLENGTH+MINMATCH) - -#define LIZARD_MAX_PRICE (1<<28) -#define LIZARD_INIT_LAST_OFFSET 0 -#define LIZARD_MAX_16BIT_OFFSET (1<<16) -#define MM_LONGOFF 16 -#define LIZARD_BLOCK_SIZE_PAD (LIZARD_BLOCK_SIZE+32) -#define LIZARD_COMPRESS_ADD_BUF (5*LIZARD_BLOCK_SIZE_PAD) -#ifndef LIZARD_NO_HUFFMAN - #define LIZARD_COMPRESS_ADD_HUF HUF_compressBound(LIZARD_BLOCK_SIZE_PAD) - #define LIZARD_HUF_BLOCK_SIZE LIZARD_BLOCK_SIZE -#else - #define LIZARD_COMPRESS_ADD_HUF 0 - #define LIZARD_HUF_BLOCK_SIZE 1 -#endif - -/* LZ4 codewords */ -#define ML_BITS_LZ4 4 -#define ML_MASK_LZ4 ((1U<= 1<<16 */ - U32 sufficientLength; /* used only by optimal parser: size of matches which is acceptable: larger == more compression, slower */ - U32 fullSearch; /* used only by optimal parser: perform full search of matches: 1 == more compression, slower */ - Lizard_parser_type parserType; - Lizard_decompress_type decompressType; -} Lizard_parameters; - - -struct Lizard_stream_s -{ - const BYTE* end; /* next block here to continue on current prefix */ - const BYTE* base; /* All index relative to this position */ - const BYTE* dictBase; /* alternate base for extDict */ - U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more dict */ - U32 nextToUpdate; /* index from which to continue dictionary update */ - U32 allocatedMemory; - int compressionLevel; - Lizard_parameters params; - U32 hashTableSize; - U32 chainTableSize; - U32* chainTable; - U32* hashTable; - int last_off; - const BYTE* off24pos; - U32 huffType; - U32 comprStreamLen; - - BYTE* huffBase; - BYTE* huffEnd; - BYTE* offset16Base; - BYTE* offset24Base; - BYTE* lenBase; - BYTE* literalsBase; - BYTE* flagsBase; - BYTE* offset16Ptr; - BYTE* offset24Ptr; - BYTE* lenPtr; - BYTE* literalsPtr; - BYTE* flagsPtr; - BYTE* offset16End; - BYTE* offset24End; - BYTE* lenEnd; - BYTE* literalsEnd; - BYTE* flagsEnd; - U32 flagFreq[256]; - U32 litFreq[256]; - U32 litSum, flagSum; - U32 litPriceSum, log2LitSum, log2FlagSum; - U32 cachedPrice; - U32 cachedLitLength; - const BYTE* cachedLiterals; - const BYTE* diffBase; - const BYTE* srcBase; - const BYTE* destBase; -}; - -struct Lizard_streamDecode_s { - const BYTE* externalDict; - size_t extDictSize; - const BYTE* prefixEnd; - size_t prefixSize; -}; - -struct Lizard_dstream_s -{ - const BYTE* offset16Ptr; - const BYTE* offset24Ptr; - const BYTE* lenPtr; - const BYTE* literalsPtr; - const BYTE* flagsPtr; - const BYTE* offset16End; - const BYTE* offset24End; - const BYTE* lenEnd; - const BYTE* literalsEnd; - const BYTE* flagsEnd; - const BYTE* diffBase; - intptr_t last_off; -}; - -typedef struct Lizard_dstream_s Lizard_dstream_t; - -/* ************************************* -* HC Pre-defined compression levels -***************************************/ -#define LIZARD_WINDOWLOG_LZ4 16 -#define LIZARD_CHAINLOG_LZ4 LIZARD_WINDOWLOG_LZ4 -#define LIZARD_HASHLOG_LZ4 18 -#define LIZARD_HASHLOG_LZ4SM 12 - -#define LIZARD_WINDOWLOG_LIZv1 22 -#define LIZARD_CHAINLOG_LIZv1 LIZARD_WINDOWLOG_LIZv1 -#define LIZARD_HASHLOG_LIZv1 18 - - - -static const Lizard_parameters Lizard_defaultParameters[LIZARD_MAX_CLEVEL+1-LIZARD_MIN_CLEVEL] = -{ - /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ - { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4SM, 0, 0, 0, 0, 0, 0, Lizard_parser_fastSmall, Lizard_coderwords_LZ4 }, // level 10 - { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_fast, Lizard_coderwords_LZ4 }, // level 11 - { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_noChain, Lizard_coderwords_LZ4 }, // level 12 - { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 2, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 13 - { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 4, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 14 - { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 8, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 15 - { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 16, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 16 - { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 256, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 17 - { LIZARD_WINDOWLOG_LZ4, LIZARD_WINDOWLOG_LZ4+1, LIZARD_HASHLOG_LZ4, 16, 16, 4, 0, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LZ4 }, // level 18 - { LIZARD_WINDOWLOG_LZ4, LIZARD_WINDOWLOG_LZ4+1, 23, 16, 256, 4, 0, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LZ4 }, // level 19 - /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ - { LIZARD_WINDOWLOG_LIZv1, 0, 14, 0, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_fastBig, Lizard_coderwords_LIZv1 }, // level 20 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 14, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 21 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 22 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 23 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 2, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 24 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 8, 4, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 25 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 8, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 26 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 128, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 27 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 28 - { 24, 25, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 29 -#ifndef LIZARD_NO_HUFFMAN - /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ - { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4SM, 0, 0, 0, 0, 0, 0, Lizard_parser_fastSmall, Lizard_coderwords_LZ4 }, // level 30 - { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_fast, Lizard_coderwords_LZ4 }, // level 31 - { LIZARD_WINDOWLOG_LZ4, 0, 14, 0, 0, 0, 0, 0, 0, Lizard_parser_noChain, Lizard_coderwords_LZ4 }, // level 32 - { LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_noChain, Lizard_coderwords_LZ4 }, // level 33 - { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 2, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 34 - { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 4, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 35 - { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 8, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 36 - { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 16, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 37 - { LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 256, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 38 - { LIZARD_WINDOWLOG_LZ4, LIZARD_WINDOWLOG_LZ4+1, 23, 16, 256, 4, 0, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LZ4 }, // level 39 - /* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */ - { LIZARD_WINDOWLOG_LIZv1, 0, 14, 0, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_fastBig, Lizard_coderwords_LIZv1 }, // level 40 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 14, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 41 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 42 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 43 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 2, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 44 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 8, 4, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 45 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 8, 4, MM_LONGOFF, 64, 0, Lizard_parser_optimalPrice, Lizard_coderwords_LIZv1 }, // level 46 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 8, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 47 - { LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 128, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 48 - { 24, 25, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 49 -#endif -// { 10, 10, 10, 0, 0, 4, 0, 0, 0, Lizard_fast }, // min values -// { 24, 24, 28, 24, 1<<24, 7, 0, 1<<24, 2, Lizard_optimal_price }, // max values -}; - - - -/*-************************************ -* Compiler Options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ -#else -# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# if defined(__GNUC__) || defined(__clang__) -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -# else -# define FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -#endif /* _MSC_VER */ - -#define LIZARD_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#if (LIZARD_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) -# define expect(expr,value) (__builtin_expect ((expr),(value)) ) -#else -# define expect(expr,value) (expr) -#endif - -#define likely(expr) expect((expr) != 0, 1) -#define unlikely(expr) expect((expr) != 0, 0) - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define ALLOCATOR(n,s) calloc(n,s) -#define FREEMEM free -#define MEM_INIT memset -#ifndef MAX - #define MAX(a,b) ((a)>(b))?(a):(b) -#endif -#ifndef MIN - #define MIN(a,b) ((a)<(b)?(a):(b)) -#endif - -#if MINMATCH == 3 - #define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr)<<8) -#else - #define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr)) -#endif - - - - -/*-************************************ -* Reading and writing into memory -**************************************/ -#define STEPSIZE sizeof(size_t) - - -MEM_STATIC void Lizard_copy8(void* dst, const void* src) -{ - memcpy(dst,src,8); -} - -/* customized variant of memcpy, which can overwrite up to 7 bytes beyond dstEnd */ -MEM_STATIC void Lizard_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) -{ - BYTE* d = (BYTE*)dstPtr; - const BYTE* s = (const BYTE*)srcPtr; - BYTE* const e = (BYTE*)dstEnd; - -#if 0 - const size_t l2 = 8 - (((size_t)d) & (sizeof(void*)-1)); - Lizard_copy8(d,s); if (d>e-9) return; - d+=l2; s+=l2; -#endif /* join to align */ - - do { Lizard_copy8(d,s); d+=8; s+=8; } while (d= 3) /* GCC Intrinsic */ - return 31 - __builtin_clz(val); -# else /* Software version */ - static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - int r; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; - return r; -# endif -} - - -/*-************************************ -* Common functions -**************************************/ -MEM_STATIC unsigned Lizard_NbCommonBytes (register size_t val) -{ - if (MEM_isLittleEndian()) { - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LIZARD_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LIZARD_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } else /* Big Endian CPU */ { - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LIZARD_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); -# else - unsigned r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LIZARD_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } - } -} - -MEM_STATIC unsigned Lizard_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) -{ - const BYTE* const pStart = pIn; - - while (likely(pIn - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOTLizard_hash4Ptr - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ - - -/* ************************************* -* Includes -***************************************/ -#include "lizard_compress.h" -#include "lizard_common.h" -#include -#include // intptr_t -#ifndef USE_LZ4_ONLY - #ifdef LIZARD_USE_TEST - #include "test/lizard_common_test.h" - #include "test/lizard_compress_test.h" - #else - #include "lizard_compress_liz.h" - #endif -#endif -#include "lizard_compress_lz4.h" -#include "entropy/huf.h" - - -/* ************************************* -* Local Macros -***************************************/ -#define DELTANEXT(p) chainTable[(p) & contentMask] -#define LIZARD_MINIMAL_HUFF_GAIN(comprSize) (comprSize + (comprSize/8) + 512) -#define LIZARD_MINIMAL_BLOCK_GAIN(comprSize) (comprSize + (comprSize/32) + 512) - - -/*-************************************ -* Local Utils -**************************************/ -int Lizard_versionNumber (void) { return LIZARD_VERSION_NUMBER; } -int Lizard_compressBound(int isize) { return LIZARD_COMPRESSBOUND(isize); } -int Lizard_sizeofState_MinLevel() { return Lizard_sizeofState(LIZARD_MIN_CLEVEL); } - - - -/* ************************************* -* Hash functions -***************************************/ -#define HASH_UPDATE_LIMIT 8 /* equal to MEM_read64 */ -static const U32 prime4bytes = 2654435761U; -static const U64 prime5bytes = 889523592379ULL; -static const U64 prime6bytes = 227718039650203ULL; -static const U64 prime7bytes = 58295818150454627ULL; - -#if MINMATCH == 3 -static const U32 prime3bytes = 506832829U; -static U32 Lizard_hash3(U32 u, U32 h) { return (u * prime3bytes) << (32-24) >> (32-h) ; } -static size_t Lizard_hash3Ptr(const void* ptr, U32 h) { return Lizard_hash3(MEM_read32(ptr), h); } -#endif - -static U32 Lizard_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } -static size_t Lizard_hash4Ptr(const void* ptr, U32 h) { return Lizard_hash4(MEM_read32(ptr), h); } - -static size_t Lizard_hash5(U64 u, U32 h) { return (size_t)((u * prime5bytes) << (64-40) >> (64-h)) ; } -static size_t Lizard_hash5Ptr(const void* p, U32 h) { return Lizard_hash5(MEM_read64(p), h); } - -static size_t Lizard_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; } -static size_t Lizard_hash6Ptr(const void* p, U32 h) { return Lizard_hash6(MEM_read64(p), h); } - -static size_t Lizard_hash7(U64 u, U32 h) { return (size_t)((u * prime7bytes) << (64-56) >> (64-h)) ; } -static size_t Lizard_hash7Ptr(const void* p, U32 h) { return Lizard_hash7(MEM_read64(p), h); } - -static size_t Lizard_hashPtr(const void* p, U32 hBits, U32 mls) -{ - switch(mls) - { - default: - case 4: return Lizard_hash4Ptr(p, hBits); - case 5: return Lizard_hash5Ptr(p, hBits); - case 6: return Lizard_hash6Ptr(p, hBits); - case 7: return Lizard_hash7Ptr(p, hBits); - } -} - - - - -/************************************** -* Internal functions -**************************************/ -/** Lizard_count_2segments() : -* can count match length with `ip` & `match` in 2 different segments. -* convention : on reaching mEnd, match count continue starting from iStart -*/ -static size_t Lizard_count_2segments(const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart) -{ - const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd); - size_t const matchLength = Lizard_count(ip, match, vEnd); - if (match + matchLength != mEnd) return matchLength; - return matchLength + Lizard_count(ip+matchLength, iStart, iEnd); -} - - -void Lizard_initBlock(Lizard_stream_t* ctx) -{ - ctx->offset16Ptr = ctx->offset16Base; - ctx->offset24Ptr = ctx->offset24Base; - ctx->lenPtr = ctx->lenBase; - ctx->literalsPtr = ctx->literalsBase; - ctx->flagsPtr = ctx->flagsBase; - ctx->last_off = LIZARD_INIT_LAST_OFFSET; /* reset last offset */ -} - - -FORCE_INLINE int Lizard_writeStream(int useHuff, Lizard_stream_t* ctx, BYTE* streamPtr, uint32_t streamLen, BYTE** op, BYTE* oend) -{ - if (useHuff && streamLen > 1024) { -#ifndef LIZARD_NO_HUFFMAN - int useHuffBuf; - if (*op + 6 > oend) { LIZARD_LOG_COMPRESS("*op[%p] + 6 > oend[%p]\n", *op, oend); return -1; } - - useHuffBuf = ((size_t)(oend - (*op + 6)) < HUF_compressBound(streamLen)); - if (useHuffBuf) { - if (streamLen > LIZARD_BLOCK_SIZE) { LIZARD_LOG_COMPRESS("streamLen[%d] > LIZARD_BLOCK_SIZE\n", streamLen); return -1; } - ctx->comprStreamLen = (U32)HUF_compress(ctx->huffBase, ctx->huffEnd - ctx->huffBase, streamPtr, streamLen); - } else { - ctx->comprStreamLen = (U32)HUF_compress(*op + 6, oend - (*op + 6), streamPtr, streamLen); - } - - if (!HUF_isError(ctx->comprStreamLen)) { - if (ctx->comprStreamLen > 0 && (LIZARD_MINIMAL_HUFF_GAIN(ctx->comprStreamLen) < streamLen)) { /* compressible */ - MEM_writeLE24(*op, streamLen); - MEM_writeLE24(*op+3, ctx->comprStreamLen); - if (useHuffBuf) { - if ((size_t)(oend - (*op + 6)) < ctx->comprStreamLen) { LIZARD_LOG_COMPRESS("*op[%p] oend[%p] comprStreamLen[%d]\n", *op, oend, (int)ctx->comprStreamLen); return -1; } - memcpy(*op + 6, ctx->huffBase, ctx->comprStreamLen); - } - *op += ctx->comprStreamLen + 6; - LIZARD_LOG_COMPRESS("HUF_compress streamLen=%d comprStreamLen=%d\n", (int)streamLen, (int)ctx->comprStreamLen); - return 1; - } else { LIZARD_LOG_COMPRESS("HUF_compress ERROR comprStreamLen=%d streamLen=%d\n", (int)ctx->comprStreamLen, (int)streamLen); } - } else { LIZARD_LOG_COMPRESS("HUF_compress ERROR %d: %s\n", (int)ctx->comprStreamLen, HUF_getErrorName(ctx->comprStreamLen)); } -#else - LIZARD_LOG_COMPRESS("compiled with LIZARD_NO_HUFFMAN\n"); - (void)ctx; - return -1; -#endif - } else ctx->comprStreamLen = 0; - - if (*op + 3 + streamLen > oend) { LIZARD_LOG_COMPRESS("*op[%p] + 3 + streamLen[%d] > oend[%p]\n", *op, streamLen, oend); return -1; } - MEM_writeLE24(*op, streamLen); - *op += 3; - memcpy(*op, streamPtr, streamLen); - *op += streamLen; - LIZARD_LOG_COMPRESS("Uncompressed streamLen=%d\n", (int)streamLen); - return 0; -} - - -int Lizard_writeBlock(Lizard_stream_t* ctx, const BYTE* ip, uint32_t inputSize, BYTE** op, BYTE* oend) -{ - int res; - uint32_t flagsLen = (uint32_t)(ctx->flagsPtr - ctx->flagsBase); - uint32_t literalsLen = (uint32_t)(ctx->literalsPtr - ctx->literalsBase); - uint32_t lenLen = (uint32_t)(ctx->lenPtr - ctx->lenBase); - uint32_t offset16Len = (uint32_t)(ctx->offset16Ptr - ctx->offset16Base); - uint32_t offset24Len = (uint32_t)(ctx->offset24Ptr - ctx->offset24Base); - uint32_t sum = flagsLen + literalsLen + lenLen + offset16Len + offset24Len; -#ifdef LIZARD_USE_LOGS - uint32_t comprFlagsLen, comprLiteralsLen; -#endif - - BYTE* start = *op; - - if ((literalsLen < WILDCOPYLENGTH) || (sum+5*3+1 > inputSize)) goto _write_uncompressed; - - *start = 0; - *op += 1; - - res = Lizard_writeStream(0, ctx, ctx->lenBase, lenLen, op, oend); - if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_LEN); - - res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_OFFSET16, ctx, ctx->offset16Base, offset16Len, op, oend); - if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_OFFSET16); - - res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_OFFSET24, ctx, ctx->offset24Base, offset24Len, op, oend); - if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_OFFSET24); - - res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_FLAGS, ctx, ctx->flagsBase, flagsLen, op, oend); - if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_FLAGS); -#ifdef LIZARD_USE_LOGS - comprFlagsLen = ctx->comprStreamLen; -#endif - - res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_LITERALS, ctx, ctx->literalsBase, literalsLen, op, oend); - if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_LITERALS); -#ifdef LIZARD_USE_LOGS - comprLiteralsLen = ctx->comprStreamLen; - sum = (int)(*op-start); -#endif - - if (LIZARD_MINIMAL_BLOCK_GAIN((uint32_t)(*op-start)) > inputSize) goto _write_uncompressed; - - LIZARD_LOG_COMPRESS("%d: total=%d block=%d flagsLen[%.2f%%]=%d comprFlagsLen[%.2f%%]=%d literalsLen[%.2f%%]=%d comprLiteralsLen[%.2f%%]=%d lenLen=%d offset16Len[%.2f%%]=%d offset24Len[%.2f%%]=%d\n", (int)(ip - ctx->srcBase), - (int)(*op - ctx->destBase), sum, (flagsLen*100.0)/sum, flagsLen, (comprFlagsLen*100.0)/sum, comprFlagsLen, (literalsLen*100.0)/sum, literalsLen, (comprLiteralsLen*100.0)/sum, comprLiteralsLen, - lenLen, (offset16Len*100.0)/sum, offset16Len, (offset24Len*100.0)/sum, offset24Len); - return 0; - -_write_uncompressed: - LIZARD_LOG_COMPRESS("%d: total=%d block=%d UNCOMPRESSED inputSize=%u outSize=%d\n", (int)(ip - ctx->srcBase), - (int)(*op - ctx->destBase), (int)(*op-start), inputSize, (int)(oend-start)); - if ((uint32_t)(oend - start) < inputSize + 4) goto _output_error; - *start = LIZARD_FLAG_UNCOMPRESSED; - *op = start + 1; - MEM_writeLE24(*op, inputSize); - *op += 3; - memcpy(*op, ip, inputSize); - *op += inputSize; - return 0; - -_output_error: - LIZARD_LOG_COMPRESS("Lizard_writeBlock ERROR size=%d/%d flagsLen=%d literalsLen=%d lenLen=%d offset16Len=%d offset24Len=%d\n", (int)(*op-start), (int)(oend-start), flagsLen, literalsLen, lenLen, offset16Len, offset24Len); - return 1; -} - - -FORCE_INLINE int Lizard_encodeSequence ( - Lizard_stream_t* ctx, - const BYTE** ip, - const BYTE** anchor, - size_t matchLength, - const BYTE* const match) -{ -#ifdef USE_LZ4_ONLY - return Lizard_encodeSequence_LZ4(ctx, ip, anchor, matchLength, match); -#else - if (ctx->params.decompressType == Lizard_coderwords_LZ4) - return Lizard_encodeSequence_LZ4(ctx, ip, anchor, matchLength, match); - - return Lizard_encodeSequence_LIZv1(ctx, ip, anchor, matchLength, match); -#endif -} - - -FORCE_INLINE int Lizard_encodeLastLiterals ( - Lizard_stream_t* ctx, - const BYTE** ip, - const BYTE** anchor) -{ - LIZARD_LOG_COMPRESS("Lizard_encodeLastLiterals Lizard_coderwords_LZ4=%d\n", ctx->params.decompressType == Lizard_coderwords_LZ4); -#ifdef USE_LZ4_ONLY - return Lizard_encodeLastLiterals_LZ4(ctx, ip, anchor); -#else - if (ctx->params.decompressType == Lizard_coderwords_LZ4) - return Lizard_encodeLastLiterals_LZ4(ctx, ip, anchor); - - return Lizard_encodeLastLiterals_LIZv1(ctx, ip, anchor); -#endif -} - - -/************************************** -* Include parsers -**************************************/ -#include "lizard_parser_hashchain.h" -#include "lizard_parser_nochain.h" -#include "lizard_parser_fast.h" -#include "lizard_parser_fastsmall.h" -#include "lizard_parser_fastbig.h" -#ifndef USE_LZ4_ONLY - #include "lizard_parser_optimal.h" - #include "lizard_parser_lowestprice.h" - #include "lizard_parser_pricefast.h" -#endif - - -int Lizard_verifyCompressionLevel(int compressionLevel) -{ - if (compressionLevel > LIZARD_MAX_CLEVEL) compressionLevel = LIZARD_MAX_CLEVEL; - if (compressionLevel < LIZARD_MIN_CLEVEL) compressionLevel = LIZARD_DEFAULT_CLEVEL; - return compressionLevel; -} - - -int Lizard_sizeofState(int compressionLevel) -{ - Lizard_parameters params; - U32 hashTableSize, chainTableSize; - - compressionLevel = Lizard_verifyCompressionLevel(compressionLevel); - params = Lizard_defaultParameters[compressionLevel - LIZARD_MIN_CLEVEL]; -// hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog3)+((size_t)1 << params.hashLog))); - hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog))); - chainTableSize = (U32)(sizeof(U32)*((size_t)1 << params.contentLog)); - - return sizeof(Lizard_stream_t) + hashTableSize + chainTableSize + LIZARD_COMPRESS_ADD_BUF + (int)LIZARD_COMPRESS_ADD_HUF; -} - - -static void Lizard_init(Lizard_stream_t* ctx, const BYTE* start) -{ - // No need to use memset() on tables as values are always bound checked -#ifdef LIZARD_RESET_MEM - MEM_INIT((void*)ctx->hashTable, 0, ctx->hashTableSize); - MEM_INIT(ctx->chainTable, 0x01, ctx->chainTableSize); -#endif - // printf("memset hashTable=%p hashEnd=%p chainTable=%p chainEnd=%p\n", ctx->hashTable, ((BYTE*)ctx->hashTable) + ctx->hashTableSize, ctx->chainTable, ((BYTE*)ctx->chainTable)+ctx->chainTableSize); - ctx->nextToUpdate = LIZARD_DICT_SIZE; - ctx->base = start - LIZARD_DICT_SIZE; - ctx->end = start; - ctx->dictBase = start - LIZARD_DICT_SIZE; - ctx->dictLimit = LIZARD_DICT_SIZE; - ctx->lowLimit = LIZARD_DICT_SIZE; - ctx->last_off = LIZARD_INIT_LAST_OFFSET; - ctx->litSum = 0; -} - - -/* if ctx==NULL memory is allocated and returned as value */ -Lizard_stream_t* Lizard_initStream(Lizard_stream_t* ctx, int compressionLevel) -{ - Lizard_parameters params; - U32 hashTableSize, chainTableSize; - void *tempPtr; - - compressionLevel = Lizard_verifyCompressionLevel(compressionLevel); - params = Lizard_defaultParameters[compressionLevel - LIZARD_MIN_CLEVEL]; -// hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog3)+((size_t)1 << params.hashLog))); - hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog))); - chainTableSize = (U32)(sizeof(U32)*((size_t)1 << params.contentLog)); - - if (!ctx) - { - ctx = (Lizard_stream_t*)malloc(sizeof(Lizard_stream_t) + hashTableSize + chainTableSize + LIZARD_COMPRESS_ADD_BUF + LIZARD_COMPRESS_ADD_HUF); - if (!ctx) { printf("ERROR: Cannot allocate %d MB (compressionLevel=%d)\n", (int)(sizeof(Lizard_stream_t) + hashTableSize + chainTableSize)>>20, compressionLevel); return 0; } - LIZARD_LOG_COMPRESS("Allocated %d MB (compressionLevel=%d)\n", (int)(sizeof(Lizard_stream_t) + hashTableSize + chainTableSize)>>20, compressionLevel); - ctx->allocatedMemory = sizeof(Lizard_stream_t) + hashTableSize + chainTableSize + LIZARD_COMPRESS_ADD_BUF + (U32)LIZARD_COMPRESS_ADD_HUF; - // printf("malloc from=%p to=%p hashTable=%p hashEnd=%p chainTable=%p chainEnd=%p\n", ctx, ((BYTE*)ctx)+sizeof(Lizard_stream_t) + hashTableSize + chainTableSize, ctx->hashTable, ((BYTE*)ctx->hashTable) + hashTableSize, ctx->chainTable, ((BYTE*)ctx->chainTable)+chainTableSize); - } - - tempPtr = ctx; - ctx->hashTable = (U32*)(tempPtr) + sizeof(Lizard_stream_t)/4; - ctx->hashTableSize = hashTableSize; - ctx->chainTable = ctx->hashTable + hashTableSize/4; - ctx->chainTableSize = chainTableSize; - ctx->params = params; - ctx->compressionLevel = (unsigned)compressionLevel; - if (compressionLevel < 30) - ctx->huffType = 0; - else - ctx->huffType = LIZARD_FLAG_LITERALS + LIZARD_FLAG_FLAGS; // + LIZARD_FLAG_OFFSET16 + LIZARD_FLAG_OFFSET24; - - ctx->literalsBase = (BYTE*)ctx->hashTable + ctx->hashTableSize + ctx->chainTableSize; - ctx->flagsBase = ctx->literalsEnd = ctx->literalsBase + LIZARD_BLOCK_SIZE_PAD; - ctx->lenBase = ctx->flagsEnd = ctx->flagsBase + LIZARD_BLOCK_SIZE_PAD; - ctx->offset16Base = ctx->lenEnd = ctx->lenBase + LIZARD_BLOCK_SIZE_PAD; - ctx->offset24Base = ctx->offset16End = ctx->offset16Base + LIZARD_BLOCK_SIZE_PAD; - ctx->huffBase = ctx->offset24End = ctx->offset24Base + LIZARD_BLOCK_SIZE_PAD; - ctx->huffEnd = ctx->huffBase + LIZARD_COMPRESS_ADD_HUF; - - return ctx; -} - - - -Lizard_stream_t* Lizard_createStream(int compressionLevel) -{ - Lizard_stream_t* ctx = Lizard_initStream(NULL, compressionLevel); - if (ctx) ctx->base = NULL; - return ctx; -} - - -/* initialization */ -Lizard_stream_t* Lizard_resetStream(Lizard_stream_t* ctx, int compressionLevel) -{ - size_t wanted = Lizard_sizeofState(compressionLevel); - - if (ctx->allocatedMemory < wanted) { - Lizard_freeStream(ctx); - ctx = Lizard_createStream(compressionLevel); - } else { - Lizard_initStream(ctx, compressionLevel); - } - - if (ctx) ctx->base = NULL; - return ctx; -} - - -int Lizard_freeStream(Lizard_stream_t* ctx) -{ - if (ctx) { - free(ctx); - } - return 0; -} - - -int Lizard_loadDict(Lizard_stream_t* Lizard_streamPtr, const char* dictionary, int dictSize) -{ - Lizard_stream_t* ctxPtr = (Lizard_stream_t*) Lizard_streamPtr; - if (dictSize > LIZARD_DICT_SIZE) { - dictionary += dictSize - LIZARD_DICT_SIZE; - dictSize = LIZARD_DICT_SIZE; - } - Lizard_init(ctxPtr, (const BYTE*)dictionary); - if (dictSize >= HASH_UPDATE_LIMIT) Lizard_Insert (ctxPtr, (const BYTE*)dictionary + (dictSize - (HASH_UPDATE_LIMIT-1))); - ctxPtr->end = (const BYTE*)dictionary + dictSize; - return dictSize; -} - - -static void Lizard_setExternalDict(Lizard_stream_t* ctxPtr, const BYTE* newBlock) -{ - if (ctxPtr->end >= ctxPtr->base + HASH_UPDATE_LIMIT) Lizard_Insert (ctxPtr, ctxPtr->end - (HASH_UPDATE_LIMIT-1)); /* Referencing remaining dictionary content */ - /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ - ctxPtr->lowLimit = ctxPtr->dictLimit; - ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); - ctxPtr->dictBase = ctxPtr->base; - ctxPtr->base = newBlock - ctxPtr->dictLimit; - ctxPtr->end = newBlock; - ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ -} - - -/* dictionary saving */ -int Lizard_saveDict (Lizard_stream_t* Lizard_streamPtr, char* safeBuffer, int dictSize) -{ - Lizard_stream_t* const ctx = (Lizard_stream_t*)Lizard_streamPtr; - int const prefixSize = (int)(ctx->end - (ctx->base + ctx->dictLimit)); - if (dictSize > LIZARD_DICT_SIZE) dictSize = LIZARD_DICT_SIZE; - if (dictSize < 4) dictSize = 0; - if (dictSize > prefixSize) dictSize = prefixSize; - memmove(safeBuffer, ctx->end - dictSize, dictSize); - { U32 const endIndex = (U32)(ctx->end - ctx->base); - ctx->end = (const BYTE*)safeBuffer + dictSize; - ctx->base = ctx->end - endIndex; - ctx->dictLimit = endIndex - dictSize; - ctx->lowLimit = endIndex - dictSize; - if (ctx->nextToUpdate < ctx->dictLimit) ctx->nextToUpdate = ctx->dictLimit; - } - return dictSize; -} - -FORCE_INLINE int Lizard_compress_generic ( - void* ctxvoid, - const char* source, - char* dest, - int inputSize, - int maxOutputSize) -{ - Lizard_stream_t* ctx = (Lizard_stream_t*) ctxvoid; - size_t dictSize = (size_t)(ctx->end - ctx->base) - ctx->dictLimit; - const BYTE* ip = (const BYTE*) source; - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + maxOutputSize; - int res; - - (void)dictSize; - LIZARD_LOG_COMPRESS("Lizard_compress_generic source=%p inputSize=%d dest=%p maxOutputSize=%d cLevel=%d dictBase=%p dictSize=%d\n", source, inputSize, dest, maxOutputSize, ctx->compressionLevel, ctx->dictBase, (int)dictSize); - *op++ = (BYTE)ctx->compressionLevel; - maxOutputSize--; // can be lower than 0 - ctx->end += inputSize; - ctx->srcBase = ctx->off24pos = ip; - ctx->destBase = (BYTE*)dest; - - while (inputSize > 0) - { - int inputPart = MIN(LIZARD_BLOCK_SIZE, inputSize); - - if (ctx->huffType) Lizard_rescaleFreqs(ctx); - Lizard_initBlock(ctx); - ctx->diffBase = ip; - - switch(ctx->params.parserType) - { - default: - case Lizard_parser_fastSmall: - res = Lizard_compress_fastSmall(ctx, ip, ip+inputPart); break; - case Lizard_parser_fast: - res = Lizard_compress_fast(ctx, ip, ip+inputPart); break; - case Lizard_parser_noChain: - res = Lizard_compress_noChain(ctx, ip, ip+inputPart); break; - case Lizard_parser_hashChain: - res = Lizard_compress_hashChain(ctx, ip, ip+inputPart); break; -#ifndef USE_LZ4_ONLY - case Lizard_parser_fastBig: - res = Lizard_compress_fastBig(ctx, ip, ip+inputPart); break; - case Lizard_parser_priceFast: - res = Lizard_compress_priceFast(ctx, ip, ip+inputPart); break; - case Lizard_parser_lowestPrice: - res = Lizard_compress_lowestPrice(ctx, ip, ip+inputPart); break; - case Lizard_parser_optimalPrice: - case Lizard_parser_optimalPriceBT: - res = Lizard_compress_optimalPrice(ctx, ip, ip+inputPart); break; -#else - case Lizard_parser_priceFast: - case Lizard_parser_lowestPrice: - case Lizard_parser_optimalPrice: - case Lizard_parser_optimalPriceBT: - res = 0; -#endif - } - - LIZARD_LOG_COMPRESS("Lizard_compress_generic res=%d inputPart=%d \n", res, inputPart); - if (res <= 0) return res; - - if (Lizard_writeBlock(ctx, ip, inputPart, &op, oend)) goto _output_error; - - ip += inputPart; - inputSize -= inputPart; - LIZARD_LOG_COMPRESS("Lizard_compress_generic in=%d out=%d\n", (int)(ip-(const BYTE*)source), (int)(op-(BYTE*)dest)); - } - - LIZARD_LOG_COMPRESS("Lizard_compress_generic total=%d\n", (int)(op-(BYTE*)dest)); - return (int)(op-(BYTE*)dest); -_output_error: - LIZARD_LOG_COMPRESS("Lizard_compress_generic ERROR\n"); - return 0; -} - - -int Lizard_compress_continue (Lizard_stream_t* ctxPtr, - const char* source, char* dest, - int inputSize, int maxOutputSize) -{ - /* auto-init if required */ - if (ctxPtr->base == NULL) Lizard_init(ctxPtr, (const BYTE*) source); - - /* Check overflow */ - if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { - size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; - if (dictSize > LIZARD_DICT_SIZE) dictSize = LIZARD_DICT_SIZE; - Lizard_loadDict((Lizard_stream_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); - } - - /* Check if blocks follow each other */ - if ((const BYTE*)source != ctxPtr->end) - Lizard_setExternalDict(ctxPtr, (const BYTE*)source); - - /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) source + inputSize; - const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; - const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; - if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) { - if (sourceEnd > dictEnd) sourceEnd = dictEnd; - ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); - if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; - } - } - - return Lizard_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize); -} - - -int Lizard_compress_extState (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) -{ - Lizard_stream_t* ctx = (Lizard_stream_t*) state; - if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - - /* initialize stream */ - Lizard_initStream(ctx, compressionLevel); - Lizard_init((Lizard_stream_t*)state, (const BYTE*)src); - - return Lizard_compress_generic (state, src, dst, srcSize, maxDstSize); -} - - -int Lizard_compress(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) -{ - int cSize; - Lizard_stream_t* statePtr = Lizard_createStream(compressionLevel); - - if (!statePtr) return 0; - cSize = Lizard_compress_extState(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); - - Lizard_freeStream(statePtr); - return cSize; -} - - -/************************************** -* Level1 functions -**************************************/ -int Lizard_compress_extState_MinLevel(void* state, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return Lizard_compress_extState(state, source, dest, inputSize, maxOutputSize, LIZARD_MIN_CLEVEL); -} - -int Lizard_compress_MinLevel(const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return Lizard_compress(source, dest, inputSize, maxOutputSize, LIZARD_MIN_CLEVEL); -} - -Lizard_stream_t* Lizard_createStream_MinLevel(void) -{ - return Lizard_createStream(LIZARD_MIN_CLEVEL); -} - -Lizard_stream_t* Lizard_resetStream_MinLevel(Lizard_stream_t* Lizard_stream) -{ - return Lizard_resetStream (Lizard_stream, LIZARD_MIN_CLEVEL); -} diff --git a/contrib/lizard/lib/lizard_compress.h b/contrib/lizard/lib/lizard_compress.h deleted file mode 100644 index d84246e2099..00000000000 --- a/contrib/lizard/lib/lizard_compress.h +++ /dev/null @@ -1,208 +0,0 @@ -/* - Lizard - Fast LZ compression algorithm - Header File - Copyright (C) 2011-2016, Yann Collet - Copyright (C) 2016-2017, Przemyslaw Skibinski - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ -#ifndef LIZARD_H_2983 -#define LIZARD_H_2983 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* - * lizard_compress.h provides block compression functions. It gives full buffer control to user. - * Block compression functions are not-enough to send information, - * since it's still necessary to provide metadata (such as compressed size), - * and each application can do it in whichever way it wants. - * For interoperability, there is Lizard frame specification (lizard_Frame_format.md). - * A library is provided to take care of it, see lizard_frame.h. -*/ - - -/*^*************************************************************** -* Export parameters -*****************************************************************/ -/* -* LIZARD_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(LIZARD_DLL_EXPORT) && (LIZARD_DLL_EXPORT==1) -# define LIZARDLIB_API __declspec(dllexport) -#elif defined(LIZARD_DLL_IMPORT) && (LIZARD_DLL_IMPORT==1) -# define LIZARDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define LIZARDLIB_API -#endif - - -/*-************************************ -* Version -**************************************/ -#define LIZARD_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LIZARD_VERSION_MINOR 0 /* for new (non-breaking) interface capabilities */ -#define LIZARD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ - -#define LIZARD_VERSION_NUMBER (LIZARD_VERSION_MAJOR *100*100 + LIZARD_VERSION_MINOR *100 + LIZARD_VERSION_RELEASE) -int Lizard_versionNumber (void); - -#define LIZARD_LIB_VERSION LIZARD_VERSION_MAJOR.LIZARD_VERSION_MINOR.LIZARD_VERSION_RELEASE -#define LIZARD_QUOTE(str) #str -#define LIZARD_EXPAND_AND_QUOTE(str) LIZARD_QUOTE(str) -#define LIZARD_VERSION_STRING LIZARD_EXPAND_AND_QUOTE(LIZARD_LIB_VERSION) -const char* Lizard_versionString (void); - -typedef struct Lizard_stream_s Lizard_stream_t; - -#define LIZARD_MIN_CLEVEL 10 /* minimum compression level */ -#ifndef LIZARD_NO_HUFFMAN - #define LIZARD_MAX_CLEVEL 49 /* maximum compression level */ -#else - #define LIZARD_MAX_CLEVEL 29 /* maximum compression level */ -#endif -#define LIZARD_DEFAULT_CLEVEL 17 - - -/*-************************************ -* Simple Functions -**************************************/ - -LIZARDLIB_API int Lizard_compress (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); - -/* -Lizard_compress() : - Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= Lizard_compressBound(sourceSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, - compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LIZARD_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails -*/ - - -/*-************************************ -* Advanced Functions -**************************************/ -#define LIZARD_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ -#define LIZARD_BLOCK_SIZE (1<<17) -#define LIZARD_BLOCK64K_SIZE (1<<16) -#define LIZARD_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LIZARD_MAX_INPUT_SIZE ? 0 : (isize) + 1 + 1 + ((isize/LIZARD_BLOCK_SIZE)+1)*4) - - -/*! -Lizard_compressBound() : - Provides the maximum size that Lizard compression may output in a "worst case" scenario (input data not compressible) - This function is primarily useful for memory allocation purposes (destination buffer size). - Macro LIZARD_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - Note that Lizard_compress() compress faster when dest buffer size is >= Lizard_compressBound(srcSize) - inputSize : max supported value is LIZARD_MAX_INPUT_SIZE - return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LIZARD_MAX_INPUT_SIZE) -*/ -LIZARDLIB_API int Lizard_compressBound(int inputSize); - - -/*! -Lizard_compress_extState() : - Same compression function, just using an externally allocated memory space to store compression state. - Use Lizard_sizeofState() to know how much memory must be allocated, - and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide it as 'void* state' to compression function. -*/ -LIZARDLIB_API int Lizard_sizeofState(int compressionLevel); - -LIZARDLIB_API int Lizard_compress_extState(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); - - - -/*-********************************************* -* Streaming Compression Functions -***********************************************/ - -/*! Lizard_createStream() will allocate and initialize an `Lizard_stream_t` structure. - * Lizard_freeStream() releases its memory. - * In the context of a DLL (liblizard), please use these methods rather than the static struct. - * They are more future proof, in case of a change of `Lizard_stream_t` size. - */ -LIZARDLIB_API Lizard_stream_t* Lizard_createStream(int compressionLevel); -LIZARDLIB_API int Lizard_freeStream (Lizard_stream_t* streamPtr); - - -/*! Lizard_resetStream() : - * Use this function to reset/reuse an allocated `Lizard_stream_t` structure - */ -LIZARDLIB_API Lizard_stream_t* Lizard_resetStream (Lizard_stream_t* streamPtr, int compressionLevel); - - -/*! Lizard_loadDict() : - * Use this function to load a static dictionary into Lizard_stream. - * Any previous data will be forgotten, only 'dictionary' will remain in memory. - * Loading a size of 0 is allowed. - * Return : dictionary size, in bytes (necessarily <= LIZARD_DICT_SIZE) - */ -LIZARDLIB_API int Lizard_loadDict (Lizard_stream_t* streamPtr, const char* dictionary, int dictSize); - - -/*! Lizard_compress_continue() : - * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. - * Important : Previous data blocks are assumed to still be present and unmodified ! - * 'dst' buffer must be already allocated. - * If maxDstSize >= Lizard_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. - */ -LIZARDLIB_API int Lizard_compress_continue (Lizard_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize); - - -/*! Lizard_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its memory location, - * save it into a safer place (char* safeBuffer). - * Note : you don't need to call Lizard_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call Lizard_compress_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. - */ -LIZARDLIB_API int Lizard_saveDict (Lizard_stream_t* streamPtr, char* safeBuffer, int dictSize); - - - - - -#if defined (__cplusplus) -} -#endif - -#endif /* LIZARD_H_2983827168210 */ diff --git a/contrib/lizard/lib/lizard_compress_liz.h b/contrib/lizard/lib/lizard_compress_liz.h deleted file mode 100644 index f531f36703a..00000000000 --- a/contrib/lizard/lib/lizard_compress_liz.h +++ /dev/null @@ -1,301 +0,0 @@ -#define LIZARD_FREQ_DIV 5 - -FORCE_INLINE void Lizard_setLog2Prices(Lizard_stream_t* ctx) -{ - ctx->log2LitSum = Lizard_highbit32(ctx->litSum+1); - ctx->log2FlagSum = Lizard_highbit32(ctx->flagSum+1); -} - - -MEM_STATIC void Lizard_rescaleFreqs(Lizard_stream_t* ctx) -{ - unsigned u; - - ctx->cachedLiterals = NULL; - ctx->cachedPrice = ctx->cachedLitLength = 0; - - ctx->litPriceSum = 0; - - if (ctx->litSum == 0) { - ctx->litSum = 2 * 256; - ctx->flagSum = 2 * 256; - - for (u=0; u < 256; u++) { - ctx->litFreq[u] = 2; - ctx->flagFreq[u] = 2; - } - } else { - ctx->litSum = 0; - ctx->flagSum = 0; - - for (u=0; u < 256; u++) { - ctx->litFreq[u] = 1 + (ctx->litFreq[u]>>LIZARD_FREQ_DIV); - ctx->litSum += ctx->litFreq[u]; - ctx->flagFreq[u] = 1 + (ctx->flagFreq[u]>>LIZARD_FREQ_DIV); - ctx->flagSum += ctx->flagFreq[u]; - } - } - - Lizard_setLog2Prices(ctx); -} - - -FORCE_INLINE int Lizard_encodeSequence_LIZv1 ( - Lizard_stream_t* ctx, - const BYTE** ip, - const BYTE** anchor, - size_t matchLength, - const BYTE* const match) -{ - U32 offset = (U32)(*ip - match); - size_t length = (size_t)(*ip - *anchor); - BYTE* token = (ctx->flagsPtr)++; - - if (length > 0 || offset < LIZARD_MAX_16BIT_OFFSET) { - /* Encode Literal length */ - // if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - length - LIZARD_LENGTH_SIZE_LIZv1(length) - WILDCOPYLENGTH)) { LIZARD_LOG_COMPRESS_LIZv1("encodeSequence overflow1\n"); return 1; } /* Check output limit */ - if (length >= MAX_SHORT_LITLEN) - { size_t len; - *token = MAX_SHORT_LITLEN; - len = length - MAX_SHORT_LITLEN; - if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; } - else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; } - else *(ctx->literalsPtr)++ = (BYTE)len; - } - else *token = (BYTE)length; - - /* Copy Literals */ - Lizard_wildCopy(ctx->literalsPtr, *anchor, (ctx->literalsPtr) + length); -#ifndef LIZARD_NO_HUFFMAN - if (ctx->huffType) { - ctx->litSum += (U32)length; - ctx->litPriceSum += (U32)(length * ctx->log2LitSum); - { U32 u; - for (u=0; u < length; u++) { - ctx->litPriceSum -= Lizard_highbit32(ctx->litFreq[ctx->literalsPtr[u]]+1); - ctx->litFreq[ctx->literalsPtr[u]]++; - } } - } -#endif - ctx->literalsPtr += length; - - - if (offset >= LIZARD_MAX_16BIT_OFFSET) { - COMPLOG_CODEWORDS_LIZv1("T32+ literal=%u match=%u offset=%d\n", (U32)length, 0, 0); - *token+=(1<huffType) { - ctx->flagFreq[*token]++; - ctx->flagSum++; - } -#endif - token = (ctx->flagsPtr)++; - } - } - - /* Encode Offset */ - if (offset >= LIZARD_MAX_16BIT_OFFSET) // 24-bit offset - { - if (matchLength < MM_LONGOFF) printf("ERROR matchLength=%d/%d\n", (int)matchLength, MM_LONGOFF), exit(1); - - // if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - 8 /*LIZARD_LENGTH_SIZE_LIZv1(length)*/)) { LIZARD_LOG_COMPRESS_LIZv1("encodeSequence overflow2\n"); return 1; } /* Check output limit */ - if (matchLength - MM_LONGOFF >= LIZARD_LAST_LONG_OFF) - { - size_t len = matchLength - MM_LONGOFF - LIZARD_LAST_LONG_OFF; - *token = LIZARD_LAST_LONG_OFF; - if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; } - else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; } - else *(ctx->literalsPtr)++ = (BYTE)len; - COMPLOG_CODEWORDS_LIZv1("T31 literal=%u match=%u offset=%d\n", 0, (U32)matchLength, offset); - } - else - { - COMPLOG_CODEWORDS_LIZv1("T0-30 literal=%u match=%u offset=%d\n", 0, (U32)matchLength, offset); - *token = (BYTE)(matchLength - MM_LONGOFF); - } - - MEM_writeLE24(ctx->offset24Ptr, offset); - ctx->offset24Ptr += 3; - ctx->last_off = offset; - ctx->off24pos = *ip; - } - else - { - COMPLOG_CODEWORDS_LIZv1("T32+ literal=%u match=%u offset=%d\n", (U32)length, (U32)matchLength, offset); - if (offset == 0) - { - *token+=(1<last_off); exit(1); } - - ctx->last_off = offset; - MEM_writeLE16(ctx->offset16Ptr, (U16)ctx->last_off); ctx->offset16Ptr += 2; - } - - /* Encode MatchLength */ - length = matchLength; - // if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - 5 /*LIZARD_LENGTH_SIZE_LIZv1(length)*/)) { LIZARD_LOG_COMPRESS_LIZv1("encodeSequence overflow2\n"); return 1; } /* Check output limit */ - if (length >= MAX_SHORT_MATCHLEN) { - *token += (BYTE)(MAX_SHORT_MATCHLEN<= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(length)); ctx->literalsPtr += 4; } - else if (length >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(length)); ctx->literalsPtr += 3; } - else *(ctx->literalsPtr)++ = (BYTE)length; - } - else *token += (BYTE)(length<huffType) { - ctx->flagFreq[*token]++; - ctx->flagSum++; - Lizard_setLog2Prices(ctx); - } -#endif - - /* Prepare next loop */ - *ip += matchLength; - *anchor = *ip; - - return 0; -} - - -FORCE_INLINE int Lizard_encodeLastLiterals_LIZv1 ( - Lizard_stream_t* ctx, - const BYTE** ip, - const BYTE** anchor) -{ - size_t length = (int)(*ip - *anchor); - (void)ctx; - - memcpy(ctx->literalsPtr, *anchor, length); - ctx->literalsPtr += length; - return 0; -} - - -#define LIZARD_PRICE_MULT 1 -#define LIZARD_GET_TOKEN_PRICE_LIZv1(token) (LIZARD_PRICE_MULT * (ctx->log2FlagSum - Lizard_highbit32(ctx->flagFreq[token]+1))) - - -FORCE_INLINE size_t Lizard_get_price_LIZv1(Lizard_stream_t* const ctx, int rep, const BYTE *ip, const BYTE *off24pos, size_t litLength, U32 offset, size_t matchLength) -{ - size_t price = 0; - BYTE token = 0; -#ifndef LIZARD_NO_HUFFMAN - const BYTE* literals = ip - litLength; - U32 u; - if ((ctx->huffType) && (ctx->params.parserType != Lizard_parser_lowestPrice)) { - if (ctx->cachedLiterals == literals && litLength >= ctx->cachedLitLength) { - size_t const additional = litLength - ctx->cachedLitLength; - const BYTE* literals2 = ctx->cachedLiterals + ctx->cachedLitLength; - price = ctx->cachedPrice + LIZARD_PRICE_MULT * additional * ctx->log2LitSum; - for (u=0; u < additional; u++) - price -= LIZARD_PRICE_MULT * Lizard_highbit32(ctx->litFreq[literals2[u]]+1); - ctx->cachedPrice = (U32)price; - ctx->cachedLitLength = (U32)litLength; - } else { - price = LIZARD_PRICE_MULT * litLength * ctx->log2LitSum; - for (u=0; u < litLength; u++) - price -= LIZARD_PRICE_MULT * Lizard_highbit32(ctx->litFreq[literals[u]]+1); - - if (litLength >= 12) { - ctx->cachedLiterals = literals; - ctx->cachedPrice = (U32)price; - ctx->cachedLitLength = (U32)litLength; - } - } - } - else - price += 8*litLength; /* Copy Literals */ -#else - price += 8*litLength; /* Copy Literals */ - (void)ip; - (void)ctx; -#endif - - (void)off24pos; - (void)rep; - - if (litLength > 0 || offset < LIZARD_MAX_16BIT_OFFSET) { - /* Encode Literal length */ - if (litLength >= MAX_SHORT_LITLEN) - { size_t len = litLength - MAX_SHORT_LITLEN; - token = MAX_SHORT_LITLEN; - if (len >= (1<<16)) price += 32; - else if (len >= 254) price += 24; - else price += 8; - } - else token = (BYTE)litLength; - - if (offset >= LIZARD_MAX_16BIT_OFFSET) { - token+=(1<huffType && ctx->params.parserType != Lizard_parser_lowestPrice) - price += LIZARD_GET_TOKEN_PRICE_LIZv1(token); - else - price += 8; - } - } - - /* Encode Offset */ - if (offset >= LIZARD_MAX_16BIT_OFFSET) { // 24-bit offset - if (matchLength < MM_LONGOFF) return LIZARD_MAX_PRICE; // error - - if (matchLength - MM_LONGOFF >= LIZARD_LAST_LONG_OFF) { - size_t len = matchLength - MM_LONGOFF - LIZARD_LAST_LONG_OFF; - token = LIZARD_LAST_LONG_OFF; - if (len >= (1<<16)) price += 32; - else if (len >= 254) price += 24; - else price += 8; - } else { - token = (BYTE)(matchLength - MM_LONGOFF); - } - - price += 24; - } else { - size_t length; - if (offset == 0) { - token+=(1<= MAX_SHORT_MATCHLEN) { - token += (BYTE)(MAX_SHORT_MATCHLEN<= (1<<16)) price += 32; - else if (length >= 254) price += 24; - else price += 8; - } - else token += (BYTE)(length< 0 || matchLength > 0) { - int offset_load = Lizard_highbit32(offset); - if (ctx->huffType) { - price += ((offset_load>=20) ? ((offset_load-19)*4) : 0); - price += 4 + (matchLength==1); - } else { - price += ((offset_load>=16) ? ((offset_load-15)*4) : 0); - price += 6 + (matchLength==1); - } - if (ctx->huffType && ctx->params.parserType != Lizard_parser_lowestPrice) - price += LIZARD_GET_TOKEN_PRICE_LIZv1(token); - else - price += 8; - } else { - if (ctx->huffType && ctx->params.parserType != Lizard_parser_lowestPrice) - price += LIZARD_GET_TOKEN_PRICE_LIZv1(token); // 1=better ratio - } - - return price; -} diff --git a/contrib/lizard/lib/lizard_compress_lz4.h b/contrib/lizard/lib/lizard_compress_lz4.h deleted file mode 100644 index 00686b6837e..00000000000 --- a/contrib/lizard/lib/lizard_compress_lz4.h +++ /dev/null @@ -1,162 +0,0 @@ -#define LIZARD_LENGTH_SIZE_LZ4(len) ((len >= (1<<16)+RUN_MASK_LZ4) ? 5 : ((len >= 254+RUN_MASK_LZ4) ? 3 : ((len >= RUN_MASK_LZ4) ? 1 : 0))) - -FORCE_INLINE int Lizard_encodeSequence_LZ4 ( - Lizard_stream_t* ctx, - const BYTE** ip, - const BYTE** anchor, - size_t matchLength, - const BYTE* const match) -{ - size_t length = (size_t)(*ip - *anchor); - BYTE* token = (ctx->flagsPtr)++; - (void) ctx; - - COMPLOG_CODEWORDS_LZ4("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); - - /* Encode Literal length */ - // if (ctx->literalsPtr > ctx->literalsEnd - length - LIZARD_LENGTH_SIZE_LZ4(length) - 2 - WILDCOPYLENGTH) { LIZARD_LOG_COMPRESS_LZ4("encodeSequence overflow1\n"); return 1; } /* Check output limit */ - if (length >= RUN_MASK_LZ4) - { size_t len = length - RUN_MASK_LZ4; - *token = RUN_MASK_LZ4; - if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; } - else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; } - else *(ctx->literalsPtr)++ = (BYTE)len; - } - else *token = (BYTE)length; - - /* Copy Literals */ - if (length > 0) { - Lizard_wildCopy(ctx->literalsPtr, *anchor, (ctx->literalsPtr) + length); -#if 0 //def LIZARD_USE_HUFFMAN - ctx->litSum += (U32)length; - ctx->litPriceSum += (U32)(length * ctx->log2LitSum); - { U32 u; - for (u=0; u < length; u++) { - ctx->litPriceSum -= Lizard_highbit32(ctx->litFreq[ctx->literalsPtr[u]]+1); - ctx->litFreq[ctx->literalsPtr[u]]++; - } } -#endif - ctx->literalsPtr += length; - } - - /* Encode Offset */ - MEM_writeLE16(ctx->literalsPtr, (U16)(*ip-match)); - ctx->literalsPtr+=2; - - /* Encode MatchLength */ - length = matchLength - MINMATCH; - // if (ctx->literalsPtr > ctx->literalsEnd - 5 /*LIZARD_LENGTH_SIZE_LZ4(length)*/) { LIZARD_LOG_COMPRESS_LZ4("encodeSequence overflow2\n"); return 1; } /* Check output limit */ - if (length >= ML_MASK_LZ4) { - *token += (BYTE)(ML_MASK_LZ4<= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(length)); ctx->literalsPtr += 4; } - else if (length >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(length)); ctx->literalsPtr += 3; } - else *(ctx->literalsPtr)++ = (BYTE)length; - } - else *token += (BYTE)(length<huffType) { - ctx->flagFreq[*token]++; - ctx->flagSum++; - Lizard_setLog2Prices(ctx); - } -#endif - - /* Prepare next loop */ - *ip += matchLength; - *anchor = *ip; - - return 0; -} - - -FORCE_INLINE int Lizard_encodeLastLiterals_LZ4 ( - Lizard_stream_t* ctx, - const BYTE** ip, - const BYTE** anchor) -{ - size_t length = (int)(*ip - *anchor); - - (void)ctx; - - memcpy(ctx->literalsPtr, *anchor, length); - ctx->literalsPtr += length; - return 0; -} - - -#define LIZARD_GET_TOKEN_PRICE_LZ4(token) (ctx->log2FlagSum - Lizard_highbit32(ctx->flagFreq[token]+1)) - -FORCE_INLINE size_t Lizard_get_price_LZ4(Lizard_stream_t* const ctx, const BYTE *ip, const size_t litLength, U32 offset, size_t matchLength) -{ - size_t price = 0; - BYTE token = 0; -#if 0 //def LIZARD_USE_HUFFMAN - const BYTE* literals = ip - litLength; - U32 u; - - if (ctx->cachedLiterals == literals && litLength >= ctx->cachedLitLength) { - size_t const additional = litLength - ctx->cachedLitLength; - const BYTE* literals2 = ctx->cachedLiterals + ctx->cachedLitLength; - price = ctx->cachedPrice + additional * ctx->log2LitSum; - for (u=0; u < additional; u++) - price -= Lizard_highbit32(ctx->litFreq[literals2[u]]+1); - ctx->cachedPrice = (U32)price; - ctx->cachedLitLength = (U32)litLength; - } else { - price = litLength * ctx->log2LitSum; - for (u=0; u < litLength; u++) - price -= Lizard_highbit32(ctx->litFreq[literals[u]]+1); - - if (litLength >= 12) { - ctx->cachedLiterals = literals; - ctx->cachedPrice = (U32)price; - ctx->cachedLitLength = (U32)litLength; - } - } -#else - price += 8*litLength; /* Copy Literals */ - (void)ip; - (void)ctx; -#endif - - /* Encode Literal length */ - if (litLength >= RUN_MASK_LZ4) { - size_t len = litLength - RUN_MASK_LZ4; - token = RUN_MASK_LZ4; - if (len >= (1<<16)) price += 32; - else if (len >= 254) price += 24; - else price += 8; - } - else token = (BYTE)litLength; - - - /* Encode MatchLength */ - if (offset) { - size_t length; - price += 16; /* Encode Offset */ - - if (offset < 8) return LIZARD_MAX_PRICE; // error - if (matchLength < MINMATCH) return LIZARD_MAX_PRICE; // error - - length = matchLength - MINMATCH; - if (length >= ML_MASK_LZ4) { - token += (BYTE)(ML_MASK_LZ4<= (1<<16)) price += 32; - else if (length >= 254) price += 24; - else price += 8; - } - else token += (BYTE)(length<huffType) { - if (offset > 0 || matchLength > 0) price += 2; - price += LIZARD_GET_TOKEN_PRICE_LZ4(token); - } else { - price += 8; // token - } - - return price; -} diff --git a/contrib/lizard/lib/lizard_decompress.c b/contrib/lizard/lib/lizard_decompress.c deleted file mode 100644 index df2eb699deb..00000000000 --- a/contrib/lizard/lib/lizard_decompress.c +++ /dev/null @@ -1,372 +0,0 @@ -/* - Lizard - Fast LZ compression algorithm - Copyright (C) 2011-2016, Yann Collet - Copyright (C) 2016-2017, Przemyslaw Skibinski - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ - - -/************************************** -* Includes -**************************************/ -//#define LIZARD_STATS 1 // 0=simple stats, 1=more, 2=full -#ifdef LIZARD_STATS - #include "test/lizard_stats.h" -#endif -#include "lizard_compress.h" -#include "lizard_decompress.h" -#include "lizard_common.h" -#include // printf -#include // intptr_t - - -/*-************************************ -* Local Structures and types -**************************************/ -typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; -typedef enum { full = 0, partial = 1 } earlyEnd_directive; - -#include "lizard_decompress_lz4.h" -#ifndef USE_LZ4_ONLY - #ifdef LIZARD_USE_TEST - #include "test/lizard_common_test.h" - #include "test/lizard_decompress_test.h" - #else - #include "lizard_decompress_liz.h" - #endif -#endif -#include "entropy/huf.h" - - -/*-***************************** -* Decompression functions -*******************************/ - -FORCE_INLINE size_t Lizard_readStream(int flag, const BYTE** ip, const BYTE* const iend, BYTE* op, BYTE* const oend, const BYTE** streamPtr, const BYTE** streamEnd, int streamFlag) -{ - if (!flag) { - if (*ip > iend - 3) return 0; - *streamPtr = *ip + 3; - *streamEnd = *streamPtr + MEM_readLE24(*ip); - if (*streamEnd < *streamPtr) return 0; - *ip = *streamEnd; -#ifdef LIZARD_STATS - uncompr_stream[streamFlag] += *streamEnd-*streamPtr; -#else - (void)streamFlag; -#endif - return 1; - } else { -#ifndef LIZARD_NO_HUFFMAN - size_t res, streamLen, comprStreamLen; - - if (*ip > iend - 6) return 0; - streamLen = MEM_readLE24(*ip); - comprStreamLen = MEM_readLE24(*ip + 3); - - if ((op > oend - streamLen) || (*ip + comprStreamLen > iend - 6)) return 0; - res = HUF_decompress(op, streamLen, *ip + 6, comprStreamLen); - if (HUF_isError(res) || (res != streamLen)) return 0; - - *ip += comprStreamLen + 6; - *streamPtr = op; - *streamEnd = *streamPtr + streamLen; -#ifdef LIZARD_STATS - compr_stream[streamFlag] += comprStreamLen + 6; - decompr_stream[streamFlag] += *streamEnd-*streamPtr; -#endif - return 1; -#else - fprintf(stderr, "compiled with LIZARD_NO_HUFFMAN\n"); - (void)op; (void)oend; - return 0; -#endif - } -} - - -FORCE_INLINE int Lizard_decompress_generic( - const char* source, - char* const dest, - int inputSize, - int outputSize, /* this value is the max size of Output Buffer. */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dest if dict == noDict */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize /* note : = 0 if noDict */ - ) -{ - /* Local Variables */ - const BYTE* ip = (const BYTE*) source, *istart = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + outputSize; - BYTE* oexit = op + targetOutputSize; - Lizard_parameters params; - Lizard_dstream_t ctx; - BYTE* decompFlagsBase, *decompOff24Base, *decompOff16Base, *decompLiteralsBase = NULL; - int res, compressionLevel; - - if (inputSize < 1) { LIZARD_LOG_DECOMPRESS("inputSize=%d outputSize=%d targetOutputSize=%d partialDecoding=%d\n", inputSize, outputSize, targetOutputSize, partialDecoding); return 0; } - - compressionLevel = *ip++; - - if (compressionLevel < LIZARD_MIN_CLEVEL || compressionLevel > LIZARD_MAX_CLEVEL) { - LIZARD_LOG_DECOMPRESS("ERROR Lizard_decompress_generic inputSize=%d compressionLevel=%d\n", inputSize, compressionLevel); - return -1; - } - - LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic ip=%p inputSize=%d targetOutputSize=%d dest=%p outputSize=%d cLevel=%d dict=%d dictSize=%d dictStart=%p partialDecoding=%d\n", ip, inputSize, targetOutputSize, dest, outputSize, compressionLevel, dict, (int)dictSize, dictStart, partialDecoding); - - decompLiteralsBase = (BYTE*)malloc(4*LIZARD_HUF_BLOCK_SIZE); - if (!decompLiteralsBase) return -1; - decompFlagsBase = decompLiteralsBase + LIZARD_HUF_BLOCK_SIZE; - decompOff24Base = decompFlagsBase + LIZARD_HUF_BLOCK_SIZE; - decompOff16Base = decompOff24Base + LIZARD_HUF_BLOCK_SIZE; - -#ifdef LIZARD_STATS - init_stats(); -#endif - (void)istart; - - while (ip < iend) - { - res = *ip++; - if (res == LIZARD_FLAG_UNCOMPRESSED) /* uncompressed */ - { - uint32_t length; - if (ip > iend - 3) { LIZARD_LOG_DECOMPRESS("UNCOMPRESSED ip[%p] > iend[%p] - 3\n", ip, iend); goto _output_error; } - length = MEM_readLE24(ip); - ip += 3; - // printf("%d: total=%d block=%d UNCOMPRESSED op=%p oexit=%p oend=%p\n", (int)(op-(BYTE*)dest) ,(int)(ip-istart), length, op, oexit, oend); - if (ip + length > iend || op + length > oend) { LIZARD_LOG_DECOMPRESS("UNCOMPRESSED ip[%p]+length[%d] > iend[%p]\n", ip, length, iend); goto _output_error; } - memcpy(op, ip, length); - op += length; - ip += length; - if ((partialDecoding) && (op >= oexit)) break; -#ifdef LIZARD_STATS - uncompr_stream[LIZARD_STREAM_UNCOMPRESSED] += length; -#endif - continue; - } - - if (res&LIZARD_FLAG_LEN) { - LIZARD_LOG_DECOMPRESS("res=%d\n", res); goto _output_error; - } - - if (ip > iend - 5*3) goto _output_error; - ctx.lenPtr = (const BYTE*)ip + 3; - ctx.lenEnd = ctx.lenPtr + MEM_readLE24(ip); - if (ctx.lenEnd < ctx.lenPtr || (ctx.lenEnd > iend - 3)) goto _output_error; -#ifdef LIZARD_STATS - uncompr_stream[LIZARD_STREAM_LEN] += ctx.lenEnd-ctx.lenPtr + 3; -#endif - ip = ctx.lenEnd; - - { size_t streamLen; -#ifdef LIZARD_USE_LOGS - const BYTE* ipos; - size_t comprFlagsLen, comprLiteralsLen, total; -#endif - streamLen = Lizard_readStream(res&LIZARD_FLAG_OFFSET16, &ip, iend, decompOff16Base, decompOff16Base + LIZARD_HUF_BLOCK_SIZE, &ctx.offset16Ptr, &ctx.offset16End, LIZARD_STREAM_OFFSET16); - if (streamLen == 0) goto _output_error; - - streamLen = Lizard_readStream(res&LIZARD_FLAG_OFFSET24, &ip, iend, decompOff24Base, decompOff24Base + LIZARD_HUF_BLOCK_SIZE, &ctx.offset24Ptr, &ctx.offset24End, LIZARD_STREAM_OFFSET24); - if (streamLen == 0) goto _output_error; - -#ifdef LIZARD_USE_LOGS - ipos = ip; - streamLen = Lizard_readStream(res&LIZARD_FLAG_FLAGS, &ip, iend, decompFlagsBase, decompFlagsBase + LIZARD_HUF_BLOCK_SIZE, &ctx.flagsPtr, &ctx.flagsEnd, LIZARD_STREAM_FLAGS); - if (streamLen == 0) goto _output_error; - streamLen = (size_t)(ctx.flagsEnd-ctx.flagsPtr); - comprFlagsLen = ((size_t)(ip - ipos) + 3 >= streamLen) ? 0 : (size_t)(ip - ipos); - ipos = ip; -#else - streamLen = Lizard_readStream(res&LIZARD_FLAG_FLAGS, &ip, iend, decompFlagsBase, decompFlagsBase + LIZARD_HUF_BLOCK_SIZE, &ctx.flagsPtr, &ctx.flagsEnd, LIZARD_STREAM_FLAGS); - if (streamLen == 0) goto _output_error; -#endif - - streamLen = Lizard_readStream(res&LIZARD_FLAG_LITERALS, &ip, iend, decompLiteralsBase, decompLiteralsBase + LIZARD_HUF_BLOCK_SIZE, &ctx.literalsPtr, &ctx.literalsEnd, LIZARD_STREAM_LITERALS); - if (streamLen == 0) goto _output_error; -#ifdef LIZARD_USE_LOGS - streamLen = (size_t)(ctx.literalsEnd-ctx.literalsPtr); - comprLiteralsLen = ((size_t)(ip - ipos) + 3 >= streamLen) ? 0 : (size_t)(ip - ipos); - total = (size_t)(ip-(ctx.lenEnd-1)); -#endif - - if (ip > iend) goto _output_error; - - LIZARD_LOG_DECOMPRESS("%d: total=%d block=%d flagsLen=%d(HUF=%d) literalsLen=%d(HUF=%d) offset16Len=%d offset24Len=%d lengthsLen=%d \n", (int)(op-(BYTE*)dest) ,(int)(ip-istart), (int)total, - (int)(ctx.flagsEnd-ctx.flagsPtr), (int)comprFlagsLen, (int)(ctx.literalsEnd-ctx.literalsPtr), (int)comprLiteralsLen, - (int)(ctx.offset16End-ctx.offset16Ptr), (int)(ctx.offset24End-ctx.offset24Ptr), (int)(ctx.lenEnd-ctx.lenPtr)); - } - - ctx.last_off = -LIZARD_INIT_LAST_OFFSET; - params = Lizard_defaultParameters[compressionLevel - LIZARD_MIN_CLEVEL]; - if (params.decompressType == Lizard_coderwords_LZ4) - res = Lizard_decompress_LZ4(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel); - else -#ifdef USE_LZ4_ONLY - res = Lizard_decompress_LZ4(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel); -#else - res = Lizard_decompress_LIZv1(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel); -#endif - LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic res=%d inputSize=%d\n", res, (int)(ctx.literalsEnd-ctx.lenEnd)); - - if (res <= 0) { free(decompLiteralsBase); return res; } - - op += res; - outputSize -= res; - if ((partialDecoding) && (op >= oexit)) break; - } - -#ifdef LIZARD_STATS - print_stats(); -#endif - - LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic total=%d\n", (int)(op-(BYTE*)dest)); - free(decompLiteralsBase); - return (int)(op-(BYTE*)dest); - -_output_error: - LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic ERROR ip=%p iend=%p\n", ip, iend); - free(decompLiteralsBase); - return -1; -} - - -int Lizard_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) -{ - return Lizard_decompress_generic(source, dest, compressedSize, maxDecompressedSize, full, 0, noDict, (BYTE*)dest, NULL, 0); -} - -int Lizard_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) -{ - return Lizard_decompress_generic(source, dest, compressedSize, maxDecompressedSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); -} - - -/*===== streaming decompression functions =====*/ - - -/* - * If you prefer dynamic allocation methods, - * Lizard_createStreamDecode() - * provides a pointer (void*) towards an initialized Lizard_streamDecode_t structure. - */ -Lizard_streamDecode_t* Lizard_createStreamDecode(void) -{ - Lizard_streamDecode_t* lizards = (Lizard_streamDecode_t*) ALLOCATOR(1, sizeof(Lizard_streamDecode_t)); - return lizards; -} - -int Lizard_freeStreamDecode (Lizard_streamDecode_t* Lizard_stream) -{ - FREEMEM(Lizard_stream); - return 0; -} - -/*! - * Lizard_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * This function is not necessary if previous data is still available where it was decoded. - * Loading a size of 0 is allowed (same effect as no dictionary). - * Return : 1 if OK, 0 if error - */ -int Lizard_setStreamDecode (Lizard_streamDecode_t* Lizard_streamDecode, const char* dictionary, int dictSize) -{ - Lizard_streamDecode_t* lizardsd = (Lizard_streamDecode_t*) Lizard_streamDecode; - lizardsd->prefixSize = (size_t) dictSize; - lizardsd->prefixEnd = (const BYTE*) dictionary + dictSize; - lizardsd->externalDict = NULL; - lizardsd->extDictSize = 0; - return 1; -} - -/* -*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where it stands using Lizard_setStreamDecode() -*/ -int Lizard_decompress_safe_continue (Lizard_streamDecode_t* Lizard_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - Lizard_streamDecode_t* lizardsd = (Lizard_streamDecode_t*) Lizard_streamDecode; - int result; - - if (lizardsd->prefixEnd == (BYTE*)dest) { - result = Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, - full, 0, usingExtDict, lizardsd->prefixEnd - lizardsd->prefixSize, lizardsd->externalDict, lizardsd->extDictSize); - if (result <= 0) return result; - lizardsd->prefixSize += result; - lizardsd->prefixEnd += result; - } else { - lizardsd->extDictSize = lizardsd->prefixSize; - lizardsd->externalDict = lizardsd->prefixEnd - lizardsd->extDictSize; - result = Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, - full, 0, usingExtDict, (BYTE*)dest, lizardsd->externalDict, lizardsd->extDictSize); - if (result <= 0) return result; - lizardsd->prefixSize = result; - lizardsd->prefixEnd = (BYTE*)dest + result; - } - - return result; -} - - -/* -Advanced decoding functions : -*_usingDict() : - These decoding functions work the same as "_continue" ones, - the dictionary must be explicitly provided within parameters -*/ - -int Lizard_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - if (dictSize==0) - return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); - if (dictStart+dictSize == dest) - { - if (dictSize >= (int)(LIZARD_DICT_SIZE - 1)) - return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, withPrefix64k, (BYTE*)dest-LIZARD_DICT_SIZE, NULL, 0); - return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); - } - return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - -/* debug function */ -int Lizard_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - diff --git a/contrib/lizard/lib/lizard_decompress.h b/contrib/lizard/lib/lizard_decompress.h deleted file mode 100644 index ad9fc8ee941..00000000000 --- a/contrib/lizard/lib/lizard_decompress.h +++ /dev/null @@ -1,152 +0,0 @@ -/* - Lizard - Fast LZ compression algorithm - Header File - Copyright (C) 2011-2016, Yann Collet - Copyright (C) 2016-2017, Przemyslaw Skibinski - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ -#ifndef LIZARD_DECOMPRESS_H_2983 -#define LIZARD_DECOMPRESS_H_2983 - -#if defined (__cplusplus) -extern "C" { -#endif - - -/*^*************************************************************** -* Export parameters -*****************************************************************/ -/* -* LIZARD_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(LIZARD_DLL_EXPORT) && (LIZARD_DLL_EXPORT==1) -# define LIZARDDLIB_API __declspec(dllexport) -#elif defined(LIZARD_DLL_IMPORT) && (LIZARD_DLL_IMPORT==1) -# define LIZARDDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define LIZARDDLIB_API -#endif - - -/*-************************************ -* Simple Functions -**************************************/ - -/* -Lizard_decompress_safe() : - compressedSize : is the precise full size of the compressed block. - maxDecompressedSize : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) - If destination buffer is not large enough, decoding will stop and output an error code (<0). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, including malicious data packets. - It never writes outside output buffer, nor reads outside input buffer. -*/ -LIZARDDLIB_API int Lizard_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); - - - -/*! -Lizard_decompress_safe_partial() : - This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. - Always control how many bytes were decoded. - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets -*/ -LIZARDDLIB_API int Lizard_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); - - - -/*-********************************************** -* Streaming Decompression Functions -************************************************/ -typedef struct Lizard_streamDecode_s Lizard_streamDecode_t; - -/* - * Lizard_streamDecode_t - * information structure to track an Lizard stream. - * init this structure content using Lizard_setStreamDecode or memset() before first use ! - * - * In the context of a DLL (liblizard) please prefer usage of construction methods below. - * They are more future proof, in case of a change of Lizard_streamDecode_t size in the future. - * Lizard_createStreamDecode will allocate and initialize an Lizard_streamDecode_t structure - * Lizard_freeStreamDecode releases its memory. - */ -LIZARDDLIB_API Lizard_streamDecode_t* Lizard_createStreamDecode(void); -LIZARDDLIB_API int Lizard_freeStreamDecode (Lizard_streamDecode_t* Lizard_stream); - -/*! Lizard_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * Setting a size of 0 is allowed (same effect as reset). - * @return : 1 if OK, 0 if error - */ -LIZARDDLIB_API int Lizard_setStreamDecode (Lizard_streamDecode_t* Lizard_streamDecode, const char* dictionary, int dictSize); - -/* -*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to LIZARD_DICT_SIZE) - In the case of a ring buffers, decoding buffer must be either : - - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - In which case, the decoding & encoding ring buffer can have any size, including small ones ( < LIZARD_DICT_SIZE). - - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including small ones ( < LIZARD_DICT_SIZE). - - _At least_ LIZARD_DICT_SIZE + 8 bytes + maxBlockSize. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including larger than decoding buffer. - Whenever these conditions are not possible, save the last LIZARD_DICT_SIZE of decoded data into a safe buffer, - and indicate where it is saved using Lizard_setStreamDecode() -*/ -LIZARDDLIB_API int Lizard_decompress_safe_continue (Lizard_streamDecode_t* Lizard_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); - - -/* -Advanced decoding functions : -*_usingDict() : - These decoding functions work the same as - a combination of Lizard_setStreamDecode() followed by Lizard_decompress_x_continue() - They are stand-alone. They don't need nor update an Lizard_streamDecode_t structure. -*/ -LIZARDDLIB_API int Lizard_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); - - -#if defined (__cplusplus) -} -#endif - -#endif /* LIZARD_DECOMPRESS_H_2983827168210 */ diff --git a/contrib/lizard/lib/lizard_decompress_liz.h b/contrib/lizard/lib/lizard_decompress_liz.h deleted file mode 100644 index 7e80eb5e01b..00000000000 --- a/contrib/lizard/lib/lizard_decompress_liz.h +++ /dev/null @@ -1,220 +0,0 @@ -/* - [0_MMMM_LLL] - 16-bit offset, 4-bit match length (4-15+), 3-bit literal length (0-7+) - [1_MMMM_LLL] - last offset, 4-bit match length (0-15+), 3-bit literal length (0-7+) - flag 31 - 24-bit offset, match length (47+), no literal length - flag 0-30 - 24-bit offset, 31 match lengths (16-46), no literal length -*/ - -/*! Lizard_decompress_LIZv1() : - * This generic decompression function cover all use cases. - * It shall be instantiated several times, using different sets of directives - * Note that it is important this generic function is really inlined, - * in order to remove useless branches during compilation optimization. - */ -FORCE_INLINE int Lizard_decompress_LIZv1( - Lizard_dstream_t* ctx, - BYTE* const dest, - int outputSize, /* this value is the max size of Output Buffer. */ - - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dest if dict == noDict */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize, /* note : = 0 if noDict */ - int compressionLevel - ) -{ - /* Local Variables */ - int inputSize = (int)(ctx->flagsEnd - ctx->flagsPtr); - const BYTE* const blockBase = ctx->flagsPtr; - const BYTE* const iend = ctx->literalsEnd; - - BYTE* op = dest; - BYTE* const oend = op + outputSize; - BYTE* cpy = NULL; - BYTE* oexit = op + targetOutputSize; - const BYTE* const lowLimit = lowPrefix - dictSize; - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - - const int checkOffset = (dictSize < (int)(LIZARD_DICT_SIZE)); - - intptr_t last_off = ctx->last_off; - intptr_t length = 0; - (void)compressionLevel; - - /* Special cases */ - if (unlikely(outputSize==0)) return ((inputSize==1) && (*ctx->flagsPtr==0)) ? 0 : -1; /* Empty output buffer */ - - /* Main Loop : decode sequences */ - while (ctx->flagsPtr < ctx->flagsEnd) { - unsigned token; - const BYTE* match; - // intptr_t litLength; - - if ((partialDecoding) && (op >= oexit)) return (int) (op-dest); - - /* get literal length */ - token = *ctx->flagsPtr++; - - if (token >= 32) - { - if ((length=(token & MAX_SHORT_LITLEN)) == MAX_SHORT_LITLEN) { - if (unlikely(ctx->literalsPtr > iend - 1)) { LIZARD_LOG_DECOMPRESS_LIZv1("1"); goto _output_error; } - length = *ctx->literalsPtr; - if unlikely(length >= 254) { - if (length == 254) { - length = MEM_readLE16(ctx->literalsPtr+1); - ctx->literalsPtr += 2; - } else { - length = MEM_readLE24(ctx->literalsPtr+1); - ctx->literalsPtr += 3; - } - } - length += MAX_SHORT_LITLEN; - ctx->literalsPtr++; - if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LIZv1("2"); goto _output_error; } /* overflow detection */ - if (unlikely((size_t)(ctx->literalsPtr+length)<(size_t)(ctx->literalsPtr))) { LIZARD_LOG_DECOMPRESS_LIZv1("3"); goto _output_error; } /* overflow detection */ - } - - /* copy literals */ - cpy = op + length; - if (unlikely(cpy > oend - WILDCOPYLENGTH || ctx->literalsPtr > iend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LIZv1("offset outside buffers\n"); goto _output_error; } /* Error : offset outside buffers */ - #if 1 - Lizard_wildCopy16(op, ctx->literalsPtr, cpy); - op = cpy; - ctx->literalsPtr += length; - #else - Lizard_copy8(op, ctx->literalsPtr); - Lizard_copy8(op+8, ctx->literalsPtr+8); - if (length > 16) - Lizard_wildCopy16(op + 16, ctx->literalsPtr + 16, cpy); - op = cpy; - ctx->literalsPtr += length; - #endif - - /* get offset */ - if (unlikely(ctx->offset16Ptr > ctx->offset16End)) { LIZARD_LOG_DECOMPRESS_LIZv1("(ctx->offset16Ptr > ctx->offset16End\n"); goto _output_error; } -#if 1 - { /* branchless */ - intptr_t new_off = MEM_readLE16(ctx->offset16Ptr); - uintptr_t not_repCode = (uintptr_t)(token >> ML_RUN_BITS) - 1; - last_off ^= not_repCode & (last_off ^ -new_off); - ctx->offset16Ptr = (BYTE*)((uintptr_t)ctx->offset16Ptr + (not_repCode & 2)); - } -#else - if ((token >> ML_RUN_BITS) == 0) - { - last_off = -(intptr_t)MEM_readLE16(ctx->offset16Ptr); - ctx->offset16Ptr += 2; - } -#endif - - /* get matchlength */ - length = (token >> RUN_BITS_LIZv1) & MAX_SHORT_MATCHLEN; - if (length == MAX_SHORT_MATCHLEN) { - if (unlikely(ctx->literalsPtr > iend - 1)) { LIZARD_LOG_DECOMPRESS_LIZv1("6"); goto _output_error; } - length = *ctx->literalsPtr; - if unlikely(length >= 254) { - if (length == 254) { - length = MEM_readLE16(ctx->literalsPtr+1); - ctx->literalsPtr += 2; - } else { - length = MEM_readLE24(ctx->literalsPtr+1); - ctx->literalsPtr += 3; - } - } - length += MAX_SHORT_MATCHLEN; - ctx->literalsPtr++; - if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LIZv1("7"); goto _output_error; } /* overflow detection */ - } - - DECOMPLOG_CODEWORDS_LIZv1("T32+ literal=%u match=%u offset=%d ipos=%d opos=%d\n", (U32)litLength, (U32)length, (int)-last_off, (U32)(ctx->flagsPtr-blockBase), (U32)(op-dest)); - } - else - if (token < LIZARD_LAST_LONG_OFF) - { - if (unlikely(ctx->offset24Ptr > ctx->offset24End - 3)) { LIZARD_LOG_DECOMPRESS_LIZv1("8"); goto _output_error; } - length = token + MM_LONGOFF; - last_off = -(intptr_t)MEM_readLE24(ctx->offset24Ptr); - ctx->offset24Ptr += 3; - DECOMPLOG_CODEWORDS_LIZv1("T0-30 literal=%u match=%u offset=%d\n", 0, (U32)length, (int)-last_off); - } - else - { - if (unlikely(ctx->literalsPtr > iend - 1)) { LIZARD_LOG_DECOMPRESS_LIZv1("9"); goto _output_error; } - length = *ctx->literalsPtr; - if unlikely(length >= 254) { - if (length == 254) { - length = MEM_readLE16(ctx->literalsPtr+1); - ctx->literalsPtr += 2; - } else { - length = MEM_readLE24(ctx->literalsPtr+1); - ctx->literalsPtr += 3; - } - } - ctx->literalsPtr++; - length += LIZARD_LAST_LONG_OFF + MM_LONGOFF; - - if (unlikely(ctx->offset24Ptr > ctx->offset24End - 3)) { LIZARD_LOG_DECOMPRESS_LIZv1("10"); goto _output_error; } - last_off = -(intptr_t)MEM_readLE24(ctx->offset24Ptr); - ctx->offset24Ptr += 3; - } - - - match = op + last_off; - if ((checkOffset) && ((unlikely((uintptr_t)(-last_off) > (uintptr_t)op) || (match < lowLimit)))) { LIZARD_LOG_DECOMPRESS_LIZv1("lowPrefix[%p]-dictSize[%d]=lowLimit[%p] match[%p]=op[%p]-last_off[%d]\n", lowPrefix, (int)dictSize, lowLimit, match, op, (int)last_off); goto _output_error; } /* Error : offset outside buffers */ - - /* check external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op + length > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LIZv1("12"); goto _output_error; } /* doesn't respect parsing restriction */ - - if (length <= (intptr_t)(lowPrefix - match)) { - /* match can be copied as a single segment from external dictionary */ - memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match encompass external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix-match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; - } else { - memcpy(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } - - /* copy match within block */ - cpy = op + length; - if (unlikely(cpy > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LIZv1("13match=%p lowLimit=%p\n", match, lowLimit); goto _output_error; } /* Error : offset outside buffers */ - Lizard_copy8(op, match); - Lizard_copy8(op+8, match+8); - if (length > 16) - Lizard_wildCopy16(op + 16, match + 16, cpy); - op = cpy; - } - - /* last literals */ - length = ctx->literalsEnd - ctx->literalsPtr; - cpy = op + length; - if ((length < 0) || (ctx->literalsPtr+length != iend) || (cpy > oend)) { LIZARD_LOG_DECOMPRESS_LIZv1("14"); goto _output_error; } /* Error : input must be consumed */ - memcpy(op, ctx->literalsPtr, length); - ctx->literalsPtr += length; - op += length; - - /* end of decoding */ - ctx->last_off = last_off; - return (int) (op-dest); /* Nb of output bytes decoded */ - - /* Overflow error detected */ -_output_error: - LIZARD_LOG_DECOMPRESS_LIZv1("_output_error=%d ctx->flagsPtr=%p blockBase=%p\n", (int) (-(ctx->flagsPtr-blockBase))-1, ctx->flagsPtr, blockBase); - LIZARD_LOG_DECOMPRESS_LIZv1("cpy=%p oend=%p ctx->literalsPtr+length[%d]=%p iend=%p\n", cpy, oend, (int)length, ctx->literalsPtr+length, iend); - return (int) (-(ctx->flagsPtr-blockBase))-1; -} diff --git a/contrib/lizard/lib/lizard_decompress_lz4.h b/contrib/lizard/lib/lizard_decompress_lz4.h deleted file mode 100644 index ffbef1cd976..00000000000 --- a/contrib/lizard/lib/lizard_decompress_lz4.h +++ /dev/null @@ -1,163 +0,0 @@ -/*! Lizard_decompress_LZ4() : - * This generic decompression function cover all use cases. - * It shall be instantiated several times, using different sets of directives - * Note that it is important this generic function is really inlined, - * in order to remove useless branches during compilation optimization. - */ -FORCE_INLINE int Lizard_decompress_LZ4( - Lizard_dstream_t* ctx, - BYTE* const dest, - int outputSize, /* this value is the max size of Output Buffer. */ - - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dest if dict == noDict */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize, /* note : = 0 if noDict */ - int compressionLevel - ) -{ - /* Local Variables */ - int inputSize = (int)(ctx->flagsEnd - ctx->flagsPtr); - const BYTE* const blockBase = ctx->flagsPtr; - const BYTE* const iend = ctx->literalsEnd; - BYTE* op = dest; - BYTE* const oend = op + outputSize; - BYTE* cpy = NULL; - BYTE* oexit = op + targetOutputSize; - const BYTE* const lowLimit = lowPrefix - dictSize; - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - - const int checkOffset = (dictSize < (int)(LIZARD_DICT_SIZE)); - - intptr_t length = 0; - (void)compressionLevel; - - /* Special cases */ - if (unlikely(outputSize==0)) return ((inputSize==1) && (*ctx->flagsPtr==0)) ? 0 : -1; /* Empty output buffer */ - - /* Main Loop : decode sequences */ - while (ctx->flagsPtr < ctx->flagsEnd) { - unsigned token; - const BYTE* match; - size_t offset; - - /* get literal length */ - token = *ctx->flagsPtr++; - if ((length=(token & RUN_MASK_LZ4)) == RUN_MASK_LZ4) { - if (unlikely(ctx->literalsPtr > iend - 5)) { LIZARD_LOG_DECOMPRESS_LZ4("0"); goto _output_error; } - length = *ctx->literalsPtr; - if unlikely(length >= 254) { - if (length == 254) { - length = MEM_readLE16(ctx->literalsPtr+1); - ctx->literalsPtr += 2; - } else { - length = MEM_readLE24(ctx->literalsPtr+1); - ctx->literalsPtr += 3; - } - } - length += RUN_MASK_LZ4; - ctx->literalsPtr++; - if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LZ4("1"); goto _output_error; } /* overflow detection */ - if (unlikely((size_t)(ctx->literalsPtr+length)<(size_t)(ctx->literalsPtr))) { LIZARD_LOG_DECOMPRESS_LZ4("2"); goto _output_error; } /* overflow detection */ - } - - /* copy literals */ - cpy = op + length; - if (unlikely(cpy > oend - WILDCOPYLENGTH || ctx->literalsPtr + length > iend - (2 + WILDCOPYLENGTH))) { LIZARD_LOG_DECOMPRESS_LZ4("offset outside buffers\n"); goto _output_error; } /* Error : offset outside buffers */ - -#if 1 - Lizard_wildCopy16(op, ctx->literalsPtr, cpy); - op = cpy; - ctx->literalsPtr += length; -#else - Lizard_copy8(op, ctx->literalsPtr); - Lizard_copy8(op+8, ctx->literalsPtr+8); - if (length > 16) - Lizard_wildCopy16(op + 16, ctx->literalsPtr + 16, cpy); - op = cpy; - ctx->literalsPtr += length; -#endif - if ((partialDecoding) && (op >= oexit)) return (int) (op-dest); - - /* get offset */ - offset = MEM_readLE16(ctx->literalsPtr); - ctx->literalsPtr += 2; - - match = op - offset; - if ((checkOffset) && (unlikely(match < lowLimit))) { LIZARD_LOG_DECOMPRESS_LZ4("lowPrefix[%p]-dictSize[%d]=lowLimit[%p] match[%p]=op[%p]-offset[%d]\n", lowPrefix, (int)dictSize, lowLimit, match, op, (int)offset); goto _output_error; } /* Error : offset outside buffers */ - - /* get matchlength */ - length = token >> RUN_BITS_LZ4; - if (length == ML_MASK_LZ4) { - if (unlikely(ctx->literalsPtr > iend - 5)) { LIZARD_LOG_DECOMPRESS_LZ4("4"); goto _output_error; } - length = *ctx->literalsPtr; - if unlikely(length >= 254) { - if (length == 254) { - length = MEM_readLE16(ctx->literalsPtr+1); - ctx->literalsPtr += 2; - } else { - length = MEM_readLE24(ctx->literalsPtr+1); - ctx->literalsPtr += 3; - } - } - length += ML_MASK_LZ4; - ctx->literalsPtr++; - if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LZ4("5"); goto _output_error; } /* overflow detection */ - } - length += MINMATCH; - - /* check external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op + length > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LZ4("6"); goto _output_error; } /* doesn't respect parsing restriction */ - - if (length <= (intptr_t)(lowPrefix - match)) { - /* match can be copied as a single segment from external dictionary */ - memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match encompass external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix-match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; - } else { - memcpy(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } - - /* copy match within block */ - cpy = op + length; - if (unlikely(cpy > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LZ4("1match=%p lowLimit=%p\n", match, lowLimit); goto _output_error; } /* Error : offset outside buffers */ - Lizard_copy8(op, match); - Lizard_copy8(op+8, match+8); - if (length > 16) - Lizard_wildCopy16(op + 16, match + 16, cpy); - op = cpy; - if ((partialDecoding) && (op >= oexit)) return (int) (op-dest); - } - - /* last literals */ - length = ctx->literalsEnd - ctx->literalsPtr; - cpy = op + length; - if ((length < 0) || (ctx->literalsPtr+length != iend) || (cpy > oend)) { LIZARD_LOG_DECOMPRESS_LZ4("9"); goto _output_error; } /* Error : input must be consumed */ - memcpy(op, ctx->literalsPtr, length); - ctx->literalsPtr += length; - op += length; - - /* end of decoding */ - return (int) (op-dest); /* Nb of output bytes decoded */ - - /* Overflow error detected */ -_output_error: - LIZARD_LOG_DECOMPRESS_LZ4("_output_error=%d ctx->flagsPtr=%p blockBase=%p\n", (int) (-(ctx->flagsPtr-blockBase))-1, ctx->flagsPtr, blockBase); - LIZARD_LOG_DECOMPRESS_LZ4("cpy=%p oend=%p ctx->literalsPtr+length[%d]=%p iend=%p\n", cpy, oend, (int)length, ctx->literalsPtr+length, iend); - return (int) (-(ctx->flagsPtr-blockBase))-1; -} diff --git a/contrib/lizard/lib/lizard_frame.c b/contrib/lizard/lib/lizard_frame.c deleted file mode 100644 index f4afbb94e0a..00000000000 --- a/contrib/lizard/lib/lizard_frame.c +++ /dev/null @@ -1,1362 +0,0 @@ -/* -Lizard auto-framing library -Copyright (C) 2011-2016, Yann Collet -Copyright (C) 2016-2017, Przemyslaw Skibinski - -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- Lizard source repository : https://github.com/inikep/lizard -*/ - -/* LizardF is a stand-alone API to create Lizard-compressed Frames -* in full conformance with specification v1.5.0 -* All related operations, including memory management, are handled by the library. -* */ - - -/*-************************************ -* Compiler Options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - - - -/*-************************************ -* Includes -**************************************/ -#include "lizard_frame_static.h" -#include "lizard_compress.h" -#include "lizard_decompress.h" -#include "lizard_common.h" /* LIZARD_DICT_SIZE */ -#define XXH_STATIC_LINKING_ONLY -#include "xxhash/xxhash.h" -#include - - - -/* unoptimized version; solves endianess & alignment issues */ -static U32 LizardF_readLE32 (const void* src) -{ - const BYTE* const srcPtr = (const BYTE*)src; - U32 value32 = srcPtr[0]; - value32 += (srcPtr[1]<<8); - value32 += (srcPtr[2]<<16); - value32 += ((U32)srcPtr[3])<<24; - return value32; -} - -static void LizardF_writeLE32 (BYTE* dstPtr, U32 value32) -{ - dstPtr[0] = (BYTE)value32; - dstPtr[1] = (BYTE)(value32 >> 8); - dstPtr[2] = (BYTE)(value32 >> 16); - dstPtr[3] = (BYTE)(value32 >> 24); -} - -static U64 LizardF_readLE64 (const BYTE* srcPtr) -{ - U64 value64 = srcPtr[0]; - value64 += ((U64)srcPtr[1]<<8); - value64 += ((U64)srcPtr[2]<<16); - value64 += ((U64)srcPtr[3]<<24); - value64 += ((U64)srcPtr[4]<<32); - value64 += ((U64)srcPtr[5]<<40); - value64 += ((U64)srcPtr[6]<<48); - value64 += ((U64)srcPtr[7]<<56); - return value64; -} - -static void LizardF_writeLE64 (BYTE* dstPtr, U64 value64) -{ - dstPtr[0] = (BYTE)value64; - dstPtr[1] = (BYTE)(value64 >> 8); - dstPtr[2] = (BYTE)(value64 >> 16); - dstPtr[3] = (BYTE)(value64 >> 24); - dstPtr[4] = (BYTE)(value64 >> 32); - dstPtr[5] = (BYTE)(value64 >> 40); - dstPtr[6] = (BYTE)(value64 >> 48); - dstPtr[7] = (BYTE)(value64 >> 56); -} - - -/*-************************************ -* Constants -**************************************/ -#define _1BIT 0x01 -#define _2BITS 0x03 -#define _3BITS 0x07 -#define _4BITS 0x0F -#define _8BITS 0xFF - -#define LIZARDF_MAGIC_SKIPPABLE_START 0x184D2A50U -#define LIZARDF_MAGICNUMBER 0x184D2206U -#define LIZARDF_BLOCKUNCOMPRESSED_FLAG 0x80000000U -#define LIZARDF_BLOCKSIZEID_DEFAULT LizardF_max128KB - -static const size_t minFHSize = 7; -static const size_t maxFHSize = 15; -static const size_t BHSize = 4; - - -/*-************************************ -* Structures and local types -**************************************/ -typedef struct LizardF_cctx_s -{ - LizardF_preferences_t prefs; - U32 version; - U32 cStage; - size_t maxBlockSize; - size_t maxBufferSize; - BYTE* tmpBuff; - BYTE* tmpIn; - size_t tmpInSize; - U64 totalInSize; - XXH32_state_t xxh; - Lizard_stream_t* lizardCtxPtr; - U32 lizardCtxLevel; /* 0: unallocated; 1: Lizard_stream_t; */ -} LizardF_cctx_t; - -typedef struct LizardF_dctx_s -{ - LizardF_frameInfo_t frameInfo; - U32 version; - U32 dStage; - U64 frameRemainingSize; - size_t maxBlockSize; - size_t maxBufferSize; - const BYTE* srcExpect; - BYTE* tmpIn; - size_t tmpInSize; - size_t tmpInTarget; - BYTE* tmpOutBuffer; - const BYTE* dict; - size_t dictSize; - BYTE* tmpOut; - size_t tmpOutSize; - size_t tmpOutStart; - XXH32_state_t xxh; - BYTE header[16]; -} LizardF_dctx_t; - - -/*-************************************ -* Error management -**************************************/ -#define LIZARDF_GENERATE_STRING(STRING) #STRING, -static const char* LizardF_errorStrings[] = { LIZARDF_LIST_ERRORS(LIZARDF_GENERATE_STRING) }; - - -unsigned LizardF_isError(LizardF_errorCode_t code) -{ - return (code > (LizardF_errorCode_t)(-LizardF_ERROR_maxCode)); -} - -const char* LizardF_getErrorName(LizardF_errorCode_t code) -{ - static const char* codeError = "Unspecified error code"; - if (LizardF_isError(code)) return LizardF_errorStrings[-(int)(code)]; - return codeError; -} - - -/*-************************************ -* Private functions -**************************************/ -static size_t LizardF_getBlockSize(unsigned blockSizeID) -{ - static const size_t blockSizes[7] = { 128 KB, 256 KB, 1 MB, 4 MB, 16 MB, 64 MB, 256 MB }; - - if (blockSizeID == 0) blockSizeID = LIZARDF_BLOCKSIZEID_DEFAULT; - blockSizeID -= 1; - if (blockSizeID >= 7) return (size_t)-LizardF_ERROR_maxBlockSize_invalid; - - return blockSizes[blockSizeID]; -} - -static LizardF_blockSizeID_t LizardF_optimalBSID(const LizardF_blockSizeID_t requestedBSID, const size_t srcSize) -{ - LizardF_blockSizeID_t proposedBSID = LizardF_max128KB; - size_t maxBlockSize; - - while (requestedBSID > proposedBSID) - { - maxBlockSize = LizardF_getBlockSize(proposedBSID); - if (srcSize <= maxBlockSize) { - return proposedBSID; - } - proposedBSID = (LizardF_blockSizeID_t)((int)proposedBSID + 1); - } - return requestedBSID; -} - - -static BYTE LizardF_headerChecksum (const void* header, size_t length) -{ - U32 xxh = XXH32(header, length, 0); - return (BYTE)(xxh >> 8); -} - - -/*-************************************ -* Simple compression functions -**************************************/ - -size_t LizardF_compressFrameBound(size_t srcSize, const LizardF_preferences_t* preferencesPtr) -{ - LizardF_preferences_t prefs; - size_t headerSize; - size_t streamSize; - - if (preferencesPtr!=NULL) prefs = *preferencesPtr; - else memset(&prefs, 0, sizeof(prefs)); - - prefs.frameInfo.blockSizeID = LizardF_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); - prefs.autoFlush = 1; - - headerSize = maxFHSize; /* header size, including magic number and frame content size*/ - streamSize = LizardF_compressBound(srcSize, &prefs); - - return headerSize + streamSize; -} - - - -/*! LizardF_compressFrame() : -* Compress an entire srcBuffer into a valid Lizard frame, as defined by specification v1.5.0, in a single step. -* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. -* You can get the minimum value of dstMaxSize by using LizardF_compressFrameBound() -* If this condition is not respected, LizardF_compressFrame() will fail (result is an errorCode) -* The LizardF_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default. -* The result of the function is the number of bytes written into dstBuffer. -* The function outputs an error code if it fails (can be tested using LizardF_isError()) -*/ -size_t LizardF_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_preferences_t* preferencesPtr) -{ - LizardF_cctx_t cctxI; - LizardF_preferences_t prefs; - LizardF_compressOptions_t options; - LizardF_errorCode_t errorCode; - BYTE* const dstStart = (BYTE*) dstBuffer; - BYTE* dstPtr = dstStart; - BYTE* const dstEnd = dstStart + dstMaxSize; - - memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */ - memset(&options, 0, sizeof(options)); - - cctxI.version = LIZARDF_VERSION; - cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ - - if (preferencesPtr!=NULL) - prefs = *preferencesPtr; - else - memset(&prefs, 0, sizeof(prefs)); - if (prefs.frameInfo.contentSize != 0) - prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ - - prefs.frameInfo.blockSizeID = LizardF_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); - prefs.autoFlush = 1; - if (srcSize <= LizardF_getBlockSize(prefs.frameInfo.blockSizeID)) - prefs.frameInfo.blockMode = LizardF_blockIndependent; /* no need for linked blocks */ - - options.stableSrc = 1; - - if (dstMaxSize < LizardF_compressFrameBound(srcSize, &prefs)) - return (size_t)-LizardF_ERROR_dstMaxSize_tooSmall; - - errorCode = LizardF_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */ - if (LizardF_isError(errorCode)) goto error; - dstPtr += errorCode; /* header size */ - - errorCode = LizardF_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); - if (LizardF_isError(errorCode)) goto error; - dstPtr += errorCode; - - errorCode = LizardF_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ - if (LizardF_isError(errorCode)) goto error; - dstPtr += errorCode; - - Lizard_freeStream(cctxI.lizardCtxPtr); - FREEMEM(cctxI.tmpBuff); - return (dstPtr - dstStart); -error: - Lizard_freeStream(cctxI.lizardCtxPtr); - FREEMEM(cctxI.tmpBuff); - return errorCode; -} - - -/*-********************************* -* Advanced compression functions -***********************************/ - -/* LizardF_createCompressionContext() : -* The first thing to do is to create a compressionContext object, which will be used in all compression operations. -* This is achieved using LizardF_createCompressionContext(), which takes as argument a version and an LizardF_preferences_t structure. -* The version provided MUST be LIZARDF_VERSION. It is intended to track potential version differences between different binaries. -* The function will provide a pointer to an allocated LizardF_compressionContext_t object. -* If the result LizardF_errorCode_t is not OK_NoError, there was an error during context creation. -* Object can release its memory using LizardF_freeCompressionContext(); -*/ -LizardF_errorCode_t LizardF_createCompressionContext(LizardF_compressionContext_t* LizardF_compressionContextPtr, unsigned version) -{ - LizardF_cctx_t* cctxPtr; - - cctxPtr = (LizardF_cctx_t*)ALLOCATOR(1, sizeof(LizardF_cctx_t)); - if (cctxPtr==NULL) return (LizardF_errorCode_t)(-LizardF_ERROR_allocation_failed); - - cctxPtr->version = version; - cctxPtr->cStage = 0; /* Next stage : write header */ - - *LizardF_compressionContextPtr = (LizardF_compressionContext_t)cctxPtr; - - return LizardF_OK_NoError; -} - - -LizardF_errorCode_t LizardF_freeCompressionContext(LizardF_compressionContext_t LizardF_compressionContext) -{ - LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)LizardF_compressionContext; - - if (cctxPtr != NULL) { /* null pointers can be safely provided to this function, like free() */ - Lizard_freeStream(cctxPtr->lizardCtxPtr); - FREEMEM(cctxPtr->tmpBuff); - FREEMEM(LizardF_compressionContext); - } - - return LizardF_OK_NoError; -} - - -/*! LizardF_compressBegin() : -* will write the frame header into dstBuffer. -* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LizardF_MAXHEADERFRAME_SIZE bytes. -* The result of the function is the number of bytes written into dstBuffer for the header -* or an error code (can be tested using LizardF_isError()) -*/ -size_t LizardF_compressBegin(LizardF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LizardF_preferences_t* preferencesPtr) -{ - LizardF_preferences_t prefNull; - LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)compressionContext; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* dstPtr = dstStart; - BYTE* headerStart; - size_t requiredBuffSize; - - if (dstMaxSize < maxFHSize) return (size_t)-LizardF_ERROR_dstMaxSize_tooSmall; - if (cctxPtr->cStage != 0) return (size_t)-LizardF_ERROR_GENERIC; - memset(&prefNull, 0, sizeof(prefNull)); - if (preferencesPtr == NULL) preferencesPtr = &prefNull; - cctxPtr->prefs = *preferencesPtr; - - /* ctx Management */ - if (cctxPtr->lizardCtxLevel == 0) { - cctxPtr->lizardCtxPtr = Lizard_createStream(cctxPtr->prefs.compressionLevel); - cctxPtr->lizardCtxLevel = 1; - } - - /* Buffer Management */ - if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LIZARDF_BLOCKSIZEID_DEFAULT; - cctxPtr->maxBlockSize = LizardF_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); - requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LizardF_blockLinked) * 2 * LIZARD_DICT_SIZE); - - if (preferencesPtr->autoFlush) - requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LizardF_blockLinked) * LIZARD_DICT_SIZE; /* just needs dict */ - - if (cctxPtr->maxBufferSize < requiredBuffSize) { - cctxPtr->maxBufferSize = requiredBuffSize; - FREEMEM(cctxPtr->tmpBuff); - cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(1, requiredBuffSize); - if (cctxPtr->tmpBuff == NULL) { printf("ERROR in LizardF_compressBegin: Cannot allocate %d MB\n", (int)(requiredBuffSize>>20)); return (size_t)-LizardF_ERROR_allocation_failed; } - } - cctxPtr->tmpIn = cctxPtr->tmpBuff; - cctxPtr->tmpInSize = 0; - XXH32_reset(&(cctxPtr->xxh), 0); - cctxPtr->lizardCtxPtr = Lizard_resetStream((Lizard_stream_t*)(cctxPtr->lizardCtxPtr), cctxPtr->prefs.compressionLevel); - if (!cctxPtr->lizardCtxPtr) return (size_t)-LizardF_ERROR_allocation_failed; - - /* Magic Number */ - LizardF_writeLE32(dstPtr, LIZARDF_MAGICNUMBER); - dstPtr += 4; - headerStart = dstPtr; - - /* FLG Byte */ - *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ - + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ - + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */ - + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)); /* Frame content size */ - /* BD Byte */ - *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); - /* Optional Frame content size field */ - if (cctxPtr->prefs.frameInfo.contentSize) { - LizardF_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize); - dstPtr += 8; - cctxPtr->totalInSize = 0; - } - /* CRC Byte */ - *dstPtr = LizardF_headerChecksum(headerStart, dstPtr - headerStart); - dstPtr++; - - cctxPtr->cStage = 1; /* header written, now request input data block */ - - return (dstPtr - dstStart); -} - - -/* LizardF_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. -* The LizardF_frameInfo_t structure is optional : -* you can provide NULL as argument, preferences will then be set to cover worst case situations. -* */ -size_t LizardF_compressBound(size_t srcSize, const LizardF_preferences_t* preferencesPtr) -{ - LizardF_preferences_t prefsNull; - memset(&prefsNull, 0, sizeof(prefsNull)); - prefsNull.frameInfo.contentChecksumFlag = LizardF_contentChecksumEnabled; /* worst case */ - { const LizardF_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; - LizardF_blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; - size_t blockSize = LizardF_getBlockSize(bid); - unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; - size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; - size_t blockInfo = 4; /* default, without block CRC option */ - size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); - - return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; - } -} - - -typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level); - -static size_t LizardF_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lizardctx, int level) -{ - /* compress one block */ - BYTE* cSizePtr = (BYTE*)dst; - U32 cSize; - cSize = (U32)compress(lizardctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); - LizardF_writeLE32(cSizePtr, cSize); - if (cSize == 0) { /* compression failed */ - cSize = (U32)srcSize; - LizardF_writeLE32(cSizePtr, cSize + LIZARDF_BLOCKUNCOMPRESSED_FLAG); - memcpy(cSizePtr+4, src, srcSize); - } - return cSize + 4; -} - - - -static int LizardF_localLizard_compress_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) -{ - (void)level; - return Lizard_compress_continue((Lizard_stream_t*)ctx, src, dst, srcSize, dstSize); -} - -static compressFunc_t LizardF_selectCompression(LizardF_blockMode_t blockMode) -{ - if (blockMode == LizardF_blockIndependent) return Lizard_compress_extState; - return LizardF_localLizard_compress_continue; -} - -static int LizardF_localSaveDict(LizardF_cctx_t* cctxPtr) -{ - return Lizard_saveDict ((Lizard_stream_t*)(cctxPtr->lizardCtxPtr), (char*)(cctxPtr->tmpBuff), LIZARD_DICT_SIZE); -} - -typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LizardF_lastBlockStatus; - -/*! LizardF_compressUpdate() : -* LizardF_compressUpdate() can be called repetitively to compress as much data as necessary. -* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. -* If this condition is not respected, LizardF_compress() will fail (result is an errorCode) -* You can get the minimum value of dstMaxSize by using LizardF_compressBound() -* The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. -* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. -* The function outputs an error code if it fails (can be tested using LizardF_isError()) -*/ -size_t LizardF_compressUpdate(LizardF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_compressOptions_t* compressOptionsPtr) -{ - LizardF_compressOptions_t cOptionsNull; - LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)compressionContext; - size_t blockSize = cctxPtr->maxBlockSize; - const BYTE* srcPtr = (const BYTE*)srcBuffer; - const BYTE* const srcEnd = srcPtr + srcSize; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* dstPtr = dstStart; - LizardF_lastBlockStatus lastBlockCompressed = notDone; - compressFunc_t compress; - - - if (cctxPtr->cStage != 1) return (size_t)-LizardF_ERROR_GENERIC; - if (dstMaxSize < LizardF_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-LizardF_ERROR_dstMaxSize_tooSmall; - memset(&cOptionsNull, 0, sizeof(cOptionsNull)); - if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; - - /* select compression function */ - compress = LizardF_selectCompression(cctxPtr->prefs.frameInfo.blockMode); - - /* complete tmp buffer */ - if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */ - size_t sizeToCopy = blockSize - cctxPtr->tmpInSize; - if (sizeToCopy > srcSize) { - /* add src to tmpIn buffer */ - memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); - srcPtr = srcEnd; - cctxPtr->tmpInSize += srcSize; - /* still needs some CRC */ - } else { - /* complete tmpIn block and then compress it */ - lastBlockCompressed = fromTmpBuffer; - memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); - srcPtr += sizeToCopy; - - dstPtr += LizardF_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lizardCtxPtr, cctxPtr->prefs.compressionLevel); - - if (cctxPtr->prefs.frameInfo.blockMode==LizardF_blockLinked) cctxPtr->tmpIn += blockSize; - cctxPtr->tmpInSize = 0; - } - } - - while ((size_t)(srcEnd - srcPtr) >= blockSize) { - /* compress full block */ - lastBlockCompressed = fromSrcBuffer; - dstPtr += LizardF_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lizardCtxPtr, cctxPtr->prefs.compressionLevel); - srcPtr += blockSize; - } - - if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { - /* compress remaining input < blockSize */ - lastBlockCompressed = fromSrcBuffer; - dstPtr += LizardF_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lizardCtxPtr, cctxPtr->prefs.compressionLevel); - srcPtr = srcEnd; - } - - /* preserve dictionary if necessary */ - if ((cctxPtr->prefs.frameInfo.blockMode==LizardF_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { - if (compressOptionsPtr->stableSrc) { - cctxPtr->tmpIn = cctxPtr->tmpBuff; - } else { - int realDictSize = LizardF_localSaveDict(cctxPtr); - if (realDictSize==0) return (size_t)-LizardF_ERROR_GENERIC; - cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; - } - } - - /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LizardF_blockLinked && lastBlockCompressed==fromTmpBuffer */ - && !(cctxPtr->prefs.autoFlush)) - { - int realDictSize = LizardF_localSaveDict(cctxPtr); - cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; - } - - /* some input data left, necessarily < blockSize */ - if (srcPtr < srcEnd) { - /* fill tmp buffer */ - size_t sizeToCopy = srcEnd - srcPtr; - memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); - cctxPtr->tmpInSize = sizeToCopy; - } - - if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LizardF_contentChecksumEnabled) - XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); - - cctxPtr->totalInSize += srcSize; - return dstPtr - dstStart; -} - - -/*! LizardF_flush() : -* Should you need to create compressed data immediately, without waiting for a block to be filled, -* you can call Lizard_flush(), which will immediately compress any remaining data stored within compressionContext. -* The result of the function is the number of bytes written into dstBuffer -* (it can be zero, this means there was no data left within compressionContext) -* The function outputs an error code if it fails (can be tested using LizardF_isError()) -* The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. -*/ -size_t LizardF_flush(LizardF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* compressOptionsPtr) -{ - LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)compressionContext; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* dstPtr = dstStart; - compressFunc_t compress; - - - if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ - if (cctxPtr->cStage != 1) return (size_t)-LizardF_ERROR_GENERIC; - if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return (size_t)-LizardF_ERROR_dstMaxSize_tooSmall; /* +8 : block header(4) + block checksum(4) */ - (void)compressOptionsPtr; /* not yet useful */ - - /* select compression function */ - compress = LizardF_selectCompression(cctxPtr->prefs.frameInfo.blockMode); - - /* compress tmp buffer */ - dstPtr += LizardF_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lizardCtxPtr, cctxPtr->prefs.compressionLevel); - if (cctxPtr->prefs.frameInfo.blockMode==LizardF_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; - cctxPtr->tmpInSize = 0; - - /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LizardF_blockLinked */ - int realDictSize = LizardF_localSaveDict(cctxPtr); - cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; - } - - return dstPtr - dstStart; -} - - -/*! LizardF_compressEnd() : -* When you want to properly finish the compressed frame, just call LizardF_compressEnd(). -* It will flush whatever data remained within compressionContext (like Lizard_flush()) -* but also properly finalize the frame, with an endMark and a checksum. -* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) -* The function outputs an error code if it fails (can be tested using LizardF_isError()) -* The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. -* compressionContext can then be used again, starting with LizardF_compressBegin(). The preferences will remain the same. -*/ -size_t LizardF_compressEnd(LizardF_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* compressOptionsPtr) -{ - LizardF_cctx_t* cctxPtr = (LizardF_cctx_t*)compressionContext; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* dstPtr = dstStart; - size_t errorCode; - - errorCode = LizardF_flush(compressionContext, dstBuffer, dstMaxSize, compressOptionsPtr); - if (LizardF_isError(errorCode)) return errorCode; - dstPtr += errorCode; - - LizardF_writeLE32(dstPtr, 0); - dstPtr+=4; /* endMark */ - - if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LizardF_contentChecksumEnabled) { - U32 xxh = XXH32_digest(&(cctxPtr->xxh)); - LizardF_writeLE32(dstPtr, xxh); - dstPtr+=4; /* content Checksum */ - } - - cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ - cctxPtr->maxBufferSize = 0; /* reuse HC context */ - - if (cctxPtr->prefs.frameInfo.contentSize) { - if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize) - return (size_t)-LizardF_ERROR_frameSize_wrong; - } - - return dstPtr - dstStart; -} - - -/*-*************************************************** -* Frame Decompression -*****************************************************/ - -/* Resource management */ - -/*! LizardF_createDecompressionContext() : -* Create a decompressionContext object, which will track all decompression operations. -* Provides a pointer to a fully allocated and initialized LizardF_decompressionContext object. -* Object can later be released using LizardF_freeDecompressionContext(). -* @return : if != 0, there was an error during context creation. -*/ -LizardF_errorCode_t LizardF_createDecompressionContext(LizardF_decompressionContext_t* LizardF_decompressionContextPtr, unsigned versionNumber) -{ - LizardF_dctx_t* const dctxPtr = (LizardF_dctx_t*)ALLOCATOR(1, sizeof(LizardF_dctx_t)); - if (dctxPtr==NULL) return (LizardF_errorCode_t)-LizardF_ERROR_GENERIC; - - dctxPtr->version = versionNumber; - *LizardF_decompressionContextPtr = (LizardF_decompressionContext_t)dctxPtr; - return LizardF_OK_NoError; -} - -LizardF_errorCode_t LizardF_freeDecompressionContext(LizardF_decompressionContext_t LizardF_decompressionContext) -{ - LizardF_errorCode_t result = LizardF_OK_NoError; - LizardF_dctx_t* const dctxPtr = (LizardF_dctx_t*)LizardF_decompressionContext; - if (dctxPtr != NULL) { /* can accept NULL input, like free() */ - result = (LizardF_errorCode_t)dctxPtr->dStage; - FREEMEM(dctxPtr->tmpIn); - FREEMEM(dctxPtr->tmpOutBuffer); - FREEMEM(dctxPtr); - } - return result; -} - - -/* ******************************************************************** */ -/* ********************* Decompression ******************************** */ -/* ******************************************************************** */ - -typedef enum { dstage_getHeader=0, dstage_storeHeader, - dstage_getCBlockSize, dstage_storeCBlockSize, - dstage_copyDirect, - dstage_getCBlock, dstage_storeCBlock, - dstage_decodeCBlock, dstage_decodeCBlock_intoDst, - dstage_decodeCBlock_intoTmp, dstage_flushOut, - dstage_getSuffix, dstage_storeSuffix, - dstage_getSFrameSize, dstage_storeSFrameSize, - dstage_skipSkippable -} dStage_t; - - -/*! LizardF_headerSize() : -* @return : size of frame header -* or an error code, which can be tested using LizardF_isError() -*/ -static size_t LizardF_headerSize(const void* src, size_t srcSize) -{ - /* minimal srcSize to determine header size */ - if (srcSize < 5) return (size_t)-LizardF_ERROR_frameHeader_incomplete; - - /* special case : skippable frames */ - if ((LizardF_readLE32(src) & 0xFFFFFFF0U) == LIZARDF_MAGIC_SKIPPABLE_START) return 8; - - /* control magic number */ - if (LizardF_readLE32(src) != LIZARDF_MAGICNUMBER) return (size_t)-LizardF_ERROR_frameType_unknown; - - /* Frame Header Size */ - { BYTE const FLG = ((const BYTE*)src)[4]; - U32 const contentSizeFlag = (FLG>>3) & _1BIT; - return contentSizeFlag ? maxFHSize : minFHSize; - } -} - - -/*! LizardF_decodeHeader() : - input : `srcVoidPtr` points at the **beginning of the frame** - output : set internal values of dctx, such as - dctxPtr->frameInfo and dctxPtr->dStage. - Also allocates internal buffers. - @return : nb Bytes read from srcVoidPtr (necessarily <= srcSize) - or an error code (testable with LizardF_isError()) -*/ -static size_t LizardF_decodeHeader(LizardF_dctx_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) -{ - BYTE FLG, BD, HC; - unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID; - size_t bufferNeeded, currentBlockSize; - size_t frameHeaderSize; - const BYTE* srcPtr = (const BYTE*)srcVoidPtr; - - /* need to decode header to get frameInfo */ - if (srcSize < minFHSize) return (size_t)-LizardF_ERROR_frameHeader_incomplete; /* minimal frame header size */ - memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo)); - - /* special case : skippable frames */ - if ((LizardF_readLE32(srcPtr) & 0xFFFFFFF0U) == LIZARDF_MAGIC_SKIPPABLE_START) { - dctxPtr->frameInfo.frameType = LizardF_skippableFrame; - if (srcVoidPtr == (void*)(dctxPtr->header)) { - dctxPtr->tmpInSize = srcSize; - dctxPtr->tmpInTarget = 8; - dctxPtr->dStage = dstage_storeSFrameSize; - return srcSize; - } else { - dctxPtr->dStage = dstage_getSFrameSize; - return 4; - } - } - - /* control magic number */ - if (LizardF_readLE32(srcPtr) != LIZARDF_MAGICNUMBER) return (size_t)-LizardF_ERROR_frameType_unknown; - dctxPtr->frameInfo.frameType = LizardF_frame; - - /* Flags */ - FLG = srcPtr[4]; - version = (FLG>>6) & _2BITS; - blockMode = (FLG>>5) & _1BIT; - blockChecksumFlag = (FLG>>4) & _1BIT; - contentSizeFlag = (FLG>>3) & _1BIT; - contentChecksumFlag = (FLG>>2) & _1BIT; - - /* Frame Header Size */ - frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize; - - if (srcSize < frameHeaderSize) { - /* not enough input to fully decode frame header */ - if (srcPtr != dctxPtr->header) - memcpy(dctxPtr->header, srcPtr, srcSize); - dctxPtr->tmpInSize = srcSize; - dctxPtr->tmpInTarget = frameHeaderSize; - dctxPtr->dStage = dstage_storeHeader; - return srcSize; - } - - BD = srcPtr[5]; - blockSizeID = (BD>>4) & _3BITS; - - /* validate */ - if (version != 1) return (size_t)-LizardF_ERROR_headerVersion_wrong; /* Version Number, only supported value */ - if (blockChecksumFlag != 0) return (size_t)-LizardF_ERROR_blockChecksum_unsupported; /* Not supported for the time being */ - if (((FLG>>0)&_2BITS) != 0) return (size_t)-LizardF_ERROR_reservedFlag_set; /* Reserved bits */ - if (((BD>>7)&_1BIT) != 0) return (size_t)-LizardF_ERROR_reservedFlag_set; /* Reserved bit */ - if (blockSizeID < 1) return (size_t)-LizardF_ERROR_maxBlockSize_invalid; /* 1-7 only supported values for the time being */ - if (((BD>>0)&_4BITS) != 0) return (size_t)-LizardF_ERROR_reservedFlag_set; /* Reserved bits */ - - /* check */ - HC = LizardF_headerChecksum(srcPtr+4, frameHeaderSize-5); - if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-LizardF_ERROR_headerChecksum_invalid; /* Bad header checksum error */ - - /* save */ - dctxPtr->frameInfo.blockMode = (LizardF_blockMode_t)blockMode; - dctxPtr->frameInfo.contentChecksumFlag = (LizardF_contentChecksum_t)contentChecksumFlag; - dctxPtr->frameInfo.blockSizeID = (LizardF_blockSizeID_t)blockSizeID; - currentBlockSize = dctxPtr->maxBlockSize; - dctxPtr->maxBlockSize = LizardF_getBlockSize(blockSizeID); - if (contentSizeFlag) - dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LizardF_readLE64(srcPtr+6); - - /* init */ - if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); - - /* alloc */ - bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LizardF_blockLinked) * 2 * LIZARD_DICT_SIZE); - if (bufferNeeded > dctxPtr->maxBufferSize || dctxPtr->maxBlockSize > currentBlockSize) { /* tmp buffers too small */ - FREEMEM(dctxPtr->tmpIn); - FREEMEM(dctxPtr->tmpOutBuffer); - dctxPtr->maxBufferSize = 0; - dctxPtr->tmpIn = (BYTE*)ALLOCATOR(1, dctxPtr->maxBlockSize); - if (dctxPtr->tmpIn == NULL) return (size_t)-LizardF_ERROR_GENERIC; - dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(1, bufferNeeded); - if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-LizardF_ERROR_GENERIC; - dctxPtr->maxBufferSize = bufferNeeded; - } - dctxPtr->tmpInSize = 0; - dctxPtr->tmpInTarget = 0; - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = 0; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; - dctxPtr->tmpOutStart = 0; - dctxPtr->tmpOutSize = 0; - - dctxPtr->dStage = dstage_getCBlockSize; - - return frameHeaderSize; -} - - -/*! LizardF_getFrameInfo() : -* Decodes frame header information, such as blockSize. -* It is optional : you could start by calling directly LizardF_decompress() instead. -* The objective is to extract header information without starting decompression, typically for allocation purposes. -* LizardF_getFrameInfo() can also be used *after* starting decompression, on a valid LizardF_decompressionContext_t. -* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). -* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) -* @return : hint of the better `srcSize` to use for next call to LizardF_decompress, -* or an error code which can be tested using LizardF_isError(). -*/ -LizardF_errorCode_t LizardF_getFrameInfo(LizardF_decompressionContext_t dCtx, LizardF_frameInfo_t* frameInfoPtr, - const void* srcBuffer, size_t* srcSizePtr) -{ - LizardF_dctx_t* dctxPtr = (LizardF_dctx_t*)dCtx; - - if (dctxPtr->dStage > dstage_storeHeader) { /* note : requires dstage_* header related to be at beginning of enum */ - /* frameInfo already decoded */ - size_t o=0, i=0; - *srcSizePtr = 0; - *frameInfoPtr = dctxPtr->frameInfo; - return LizardF_decompress(dCtx, NULL, &o, NULL, &i, NULL); /* returns : recommended nb of bytes for LizardF_decompress() */ - } else { - size_t nextSrcSize, o=0; - size_t const hSize = LizardF_headerSize(srcBuffer, *srcSizePtr); - if (LizardF_isError(hSize)) { *srcSizePtr=0; return hSize; } - if (*srcSizePtr < hSize) { *srcSizePtr=0; return (size_t)-LizardF_ERROR_frameHeader_incomplete; } - - *srcSizePtr = hSize; - nextSrcSize = LizardF_decompress(dCtx, NULL, &o, srcBuffer, srcSizePtr, NULL); - if (dctxPtr->dStage <= dstage_storeHeader) return (size_t)-LizardF_ERROR_frameHeader_incomplete; /* should not happen, already checked */ - *frameInfoPtr = dctxPtr->frameInfo; - return nextSrcSize; - } -} - - -/* trivial redirector, for common prototype */ -static int LizardF_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) -{ - (void)dictStart; (void)dictSize; - return Lizard_decompress_safe (source, dest, compressedSize, maxDecompressedSize); -} - - -static void LizardF_updateDict(LizardF_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) -{ - if (dctxPtr->dictSize==0) - dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ - - if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) { /* dictionary continuity */ - dctxPtr->dictSize += dstSize; - return; - } - - if (dstPtr - dstPtr0 + dstSize >= LIZARD_DICT_SIZE) { /* dstBuffer large enough to become dictionary */ - dctxPtr->dict = (const BYTE*)dstPtr0; - dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize; - return; - } - - if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer)) { - /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */ - dctxPtr->dictSize += dstSize; - return; - } - - if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ - size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - size_t copySize = LIZARD_DICT_SIZE - dctxPtr->tmpOutSize; - const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; - if (dctxPtr->tmpOutSize > LIZARD_DICT_SIZE) copySize = 0; - if (copySize > preserveSize) copySize = preserveSize; - - memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); - - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize; - return; - } - - if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { /* copy dst into tmp to complete dict */ - if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) { /* tmp buffer not large enough */ - size_t preserveSize = LIZARD_DICT_SIZE - dstSize; /* note : dstSize < LIZARD_DICT_SIZE */ - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); - dctxPtr->dictSize = preserveSize; - } - memcpy(dctxPtr->tmpOutBuffer + dctxPtr->dictSize, dstPtr, dstSize); - dctxPtr->dictSize += dstSize; - return; - } - - /* join dict & dest into tmp */ - { size_t preserveSize = LIZARD_DICT_SIZE - dstSize; /* note : dstSize < LIZARD_DICT_SIZE */ - if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize; - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); - memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = preserveSize + dstSize; - } -} - - - -/*! LizardF_decompress() : -* Call this function repetitively to regenerate data compressed within srcBuffer. -* The function will attempt to decode *srcSizePtr from srcBuffer, into dstBuffer of maximum size *dstSizePtr. -* -* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). -* -* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). -* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. -* You will have to call it again, continuing from where it stopped. -* -* The function result is an hint of the better srcSize to use for next call to LizardF_decompress. -* Basically, it's the size of the current (or remaining) compressed block + header of next block. -* Respecting the hint provides some boost to performance, since it allows less buffer shuffling. -* Note that this is just a hint, you can always provide any srcSize you want. -* When a frame is fully decoded, the function result will be 0. -* If decompression failed, function result is an error code which can be tested using LizardF_isError(). -*/ -size_t LizardF_decompress(LizardF_decompressionContext_t decompressionContext, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const LizardF_decompressOptions_t* decompressOptionsPtr) -{ - LizardF_dctx_t* dctxPtr = (LizardF_dctx_t*)decompressionContext; - LizardF_decompressOptions_t optionsNull; - const BYTE* const srcStart = (const BYTE*)srcBuffer; - const BYTE* const srcEnd = srcStart + *srcSizePtr; - const BYTE* srcPtr = srcStart; - BYTE* const dstStart = (BYTE*)dstBuffer; - BYTE* const dstEnd = dstStart + *dstSizePtr; - BYTE* dstPtr = dstStart; - const BYTE* selectedIn = NULL; - unsigned doAnotherStage = 1; - size_t nextSrcSizeHint = 1; - - - memset(&optionsNull, 0, sizeof(optionsNull)); - if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; - *srcSizePtr = 0; - *dstSizePtr = 0; - - /* expect to continue decoding src buffer where it left previously */ - if (dctxPtr->srcExpect != NULL) { - if (srcStart != dctxPtr->srcExpect) return (size_t)-LizardF_ERROR_srcPtr_wrong; - } - - /* programmed as a state machine */ - - while (doAnotherStage) { - - switch(dctxPtr->dStage) - { - - case dstage_getHeader: - if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */ - LizardF_errorCode_t const hSize = LizardF_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr); - if (LizardF_isError(hSize)) return hSize; - srcPtr += hSize; - break; - } - dctxPtr->tmpInSize = 0; - dctxPtr->tmpInTarget = minFHSize; /* minimum to attempt decode */ - dctxPtr->dStage = dstage_storeHeader; - /* fallthrough */ - - case dstage_storeHeader: - { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - dctxPtr->tmpInSize += sizeToCopy; - srcPtr += sizeToCopy; - if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { - nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ - doAnotherStage = 0; /* not enough src data, ask for some more */ - break; - } - { LizardF_errorCode_t const hSize = LizardF_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); - if (LizardF_isError(hSize)) return hSize; - } - break; - } - - case dstage_getCBlockSize: - if ((size_t)(srcEnd - srcPtr) >= BHSize) { - selectedIn = srcPtr; - srcPtr += BHSize; - } else { - /* not enough input to read cBlockSize field */ - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeCBlockSize; - } - - if (dctxPtr->dStage == dstage_storeCBlockSize) /* can be skipped */ - case dstage_storeCBlockSize: - { - size_t sizeToCopy = BHSize - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < BHSize) { /* not enough input to get full cBlockSize; wait for more */ - nextSrcSizeHint = BHSize - dctxPtr->tmpInSize; - doAnotherStage = 0; - break; - } - selectedIn = dctxPtr->tmpIn; - } - - /* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */ - { size_t const nextCBlockSize = LizardF_readLE32(selectedIn) & 0x7FFFFFFFU; - if (nextCBlockSize==0) { /* frameEnd signal, no more CBlock */ - dctxPtr->dStage = dstage_getSuffix; - break; - } - if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-LizardF_ERROR_GENERIC; /* invalid cBlockSize */ - dctxPtr->tmpInTarget = nextCBlockSize; - if (LizardF_readLE32(selectedIn) & LIZARDF_BLOCKUNCOMPRESSED_FLAG) { - dctxPtr->dStage = dstage_copyDirect; - break; - } - dctxPtr->dStage = dstage_getCBlock; - if (dstPtr==dstEnd) { - nextSrcSizeHint = nextCBlockSize + BHSize; - doAnotherStage = 0; - } - break; - } - - case dstage_copyDirect: /* uncompressed block */ - { size_t sizeToCopy = dctxPtr->tmpInTarget; - if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */ - if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; - memcpy(dstPtr, srcPtr, sizeToCopy); - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy); - if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy; - - /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==LizardF_blockLinked) - LizardF_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); - - srcPtr += sizeToCopy; - dstPtr += sizeToCopy; - if (sizeToCopy == dctxPtr->tmpInTarget) { /* all copied */ - dctxPtr->dStage = dstage_getCBlockSize; - break; - } - dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */ - nextSrcSizeHint = dctxPtr->tmpInTarget + BHSize; - doAnotherStage = 0; - break; - } - - case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */ - if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) { - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeCBlock; - break; - } - selectedIn = srcPtr; - srcPtr += dctxPtr->tmpInTarget; - dctxPtr->dStage = dstage_decodeCBlock; - break; - - case dstage_storeCBlock: - { size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - dctxPtr->tmpInSize += sizeToCopy; - srcPtr += sizeToCopy; - if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* need more input */ - nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; - doAnotherStage=0; - break; - } - selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_decodeCBlock; - } - /* fallthrough */ - - case dstage_decodeCBlock: - if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */ - dctxPtr->dStage = dstage_decodeCBlock_intoTmp; - else - dctxPtr->dStage = dstage_decodeCBlock_intoDst; - break; - - case dstage_decodeCBlock_intoDst: - { int (*decoder)(const char*, char*, int, int, const char*, int); - int decodedSize; - - if (dctxPtr->frameInfo.blockMode == LizardF_blockLinked) - decoder = Lizard_decompress_safe_usingDict; - else - decoder = LizardF_decompress_safe; - - decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) return (size_t)-LizardF_ERROR_GENERIC; /* decompression failed */ - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); - if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; - - /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==LizardF_blockLinked) - LizardF_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); - - dstPtr += decodedSize; - dctxPtr->dStage = dstage_getCBlockSize; - break; - } - - case dstage_decodeCBlock_intoTmp: - /* not enough place into dst : decode into tmpOut */ - { int (*decoder)(const char*, char*, int, int, const char*, int); - int decodedSize; - - if (dctxPtr->frameInfo.blockMode == LizardF_blockLinked) - decoder = Lizard_decompress_safe_usingDict; - else - decoder = LizardF_decompress_safe; - - /* ensure enough place for tmpOut */ - if (dctxPtr->frameInfo.blockMode == LizardF_blockLinked) { - if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { - if (dctxPtr->dictSize > 2 * LIZARD_DICT_SIZE) { - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - LIZARD_DICT_SIZE, LIZARD_DICT_SIZE); - dctxPtr->dictSize = LIZARD_DICT_SIZE; - } - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize; - } else { /* dict not within tmp */ - size_t reservedDictSpace = dctxPtr->dictSize; - if (reservedDictSpace > LIZARD_DICT_SIZE) reservedDictSpace = LIZARD_DICT_SIZE; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace; - } - } - - /* Decode */ - decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) return (size_t)-LizardF_ERROR_decompressionFailed; /* decompression failed */ - if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); - if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; - dctxPtr->tmpOutSize = decodedSize; - dctxPtr->tmpOutStart = 0; - dctxPtr->dStage = dstage_flushOut; - break; - } - - case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ - { size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart; - if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr; - memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); - - /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==LizardF_blockLinked) - LizardF_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); - - dctxPtr->tmpOutStart += sizeToCopy; - dstPtr += sizeToCopy; - - /* end of flush ? */ - if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) { - dctxPtr->dStage = dstage_getCBlockSize; - break; - } - nextSrcSizeHint = BHSize; - doAnotherStage = 0; /* still some data to flush */ - break; - } - - case dstage_getSuffix: - { size_t const suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; - if (dctxPtr->frameRemainingSize) return (size_t)-LizardF_ERROR_frameSize_wrong; /* incorrect frame size decoded */ - if (suffixSize == 0) { /* frame completed */ - nextSrcSizeHint = 0; - dctxPtr->dStage = dstage_getHeader; - doAnotherStage = 0; - break; - } - if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */ - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeSuffix; - } else { - selectedIn = srcPtr; - srcPtr += 4; - } - } - - if (dctxPtr->dStage == dstage_storeSuffix) /* can be skipped */ - case dstage_storeSuffix: - { - size_t sizeToCopy = 4 - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < 4) { /* not enough input to read complete suffix */ - nextSrcSizeHint = 4 - dctxPtr->tmpInSize; - doAnotherStage=0; - break; - } - selectedIn = dctxPtr->tmpIn; - } - - /* case dstage_checkSuffix: */ /* no direct call, to avoid scan-build warning */ - { U32 const readCRC = LizardF_readLE32(selectedIn); - U32 const resultCRC = XXH32_digest(&(dctxPtr->xxh)); - if (readCRC != resultCRC) return (size_t)-LizardF_ERROR_contentChecksum_invalid; - nextSrcSizeHint = 0; - dctxPtr->dStage = dstage_getHeader; - doAnotherStage = 0; - break; - } - - case dstage_getSFrameSize: - if ((srcEnd - srcPtr) >= 4) { - selectedIn = srcPtr; - srcPtr += 4; - } else { - /* not enough input to read cBlockSize field */ - dctxPtr->tmpInSize = 4; - dctxPtr->tmpInTarget = 8; - dctxPtr->dStage = dstage_storeSFrameSize; - } - - if (dctxPtr->dStage == dstage_storeSFrameSize) - case dstage_storeSFrameSize: - { - size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; - memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); - srcPtr += sizeToCopy; - dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { /* not enough input to get full sBlockSize; wait for more */ - nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; - doAnotherStage = 0; - break; - } - selectedIn = dctxPtr->header + 4; - } - - /* case dstage_decodeSFrameSize: */ /* no direct access */ - { size_t const SFrameSize = LizardF_readLE32(selectedIn); - dctxPtr->frameInfo.contentSize = SFrameSize; - dctxPtr->tmpInTarget = SFrameSize; - dctxPtr->dStage = dstage_skipSkippable; - break; - } - - case dstage_skipSkippable: - { size_t skipSize = dctxPtr->tmpInTarget; - if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr; - srcPtr += skipSize; - dctxPtr->tmpInTarget -= skipSize; - doAnotherStage = 0; - nextSrcSizeHint = dctxPtr->tmpInTarget; - if (nextSrcSizeHint) break; - dctxPtr->dStage = dstage_getHeader; - break; - } - } - } - - /* preserve dictionary within tmp if necessary */ - if ( (dctxPtr->frameInfo.blockMode==LizardF_blockLinked) - &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) - &&(!decompressOptionsPtr->stableDst) - &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) - ) - { - if (dctxPtr->dStage == dstage_flushOut) { - size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - size_t copySize = LIZARD_DICT_SIZE - dctxPtr->tmpOutSize; - const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; - if (dctxPtr->tmpOutSize > LIZARD_DICT_SIZE) copySize = 0; - if (copySize > preserveSize) copySize = preserveSize; - - memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); - - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart; - } else { - size_t newDictSize = dctxPtr->dictSize; - const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; - if ((newDictSize) > LIZARD_DICT_SIZE) newDictSize = LIZARD_DICT_SIZE; - - memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); - - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = newDictSize; - dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize; - } - } - - /* require function to be called again from position where it stopped */ - if (srcPtrsrcExpect = srcPtr; - else - dctxPtr->srcExpect = NULL; - - *srcSizePtr = (srcPtr - srcStart); - *dstSizePtr = (dstPtr - dstStart); - return nextSrcSizeHint; -} diff --git a/contrib/lizard/lib/lizard_frame.h b/contrib/lizard/lib/lizard_frame.h deleted file mode 100644 index 7cdd20063c5..00000000000 --- a/contrib/lizard/lib/lizard_frame.h +++ /dev/null @@ -1,303 +0,0 @@ -/* - Lizard auto-framing library - Header File - Copyright (C) 2011-2015, Yann Collet - Copyright (C) 2016-2017, Przemyslaw Skibinski - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ - -/* LizardF is a stand-alone API to create Lizard-compressed frames - * conformant with specification v1.5.1. - * All related operations, including memory management, are handled internally by the library. - * You don't need lizard_compress.h when using lizard_frame.h. - * */ - -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - -/*-************************************ -* Includes -**************************************/ -#include /* size_t */ - - -/*-************************************ -* Error management -**************************************/ -typedef size_t LizardF_errorCode_t; - -unsigned LizardF_isError(LizardF_errorCode_t code); -const char* LizardF_getErrorName(LizardF_errorCode_t code); /* return error code string; useful for debugging */ - - -/*-************************************ -* Frame compression types -**************************************/ -//#define LIZARDF_DISABLE_OBSOLETE_ENUMS -#ifndef LIZARDF_DISABLE_OBSOLETE_ENUMS -# define LIZARDF_OBSOLETE_ENUM(x) ,x -#else -# define LIZARDF_OBSOLETE_ENUM(x) -#endif - -typedef enum { - LizardF_default=0, - LizardF_max128KB=1, - LizardF_max256KB=2, - LizardF_max1MB=3, - LizardF_max4MB=4, - LizardF_max16MB=5, - LizardF_max64MB=6, - LizardF_max256MB=7 -} LizardF_blockSizeID_t; - -typedef enum { - LizardF_blockLinked=0, - LizardF_blockIndependent - LIZARDF_OBSOLETE_ENUM(blockLinked = LizardF_blockLinked) - LIZARDF_OBSOLETE_ENUM(blockIndependent = LizardF_blockIndependent) -} LizardF_blockMode_t; - -typedef enum { - LizardF_noContentChecksum=0, - LizardF_contentChecksumEnabled - LIZARDF_OBSOLETE_ENUM(noContentChecksum = LizardF_noContentChecksum) - LIZARDF_OBSOLETE_ENUM(contentChecksumEnabled = LizardF_contentChecksumEnabled) -} LizardF_contentChecksum_t; - -typedef enum { - LizardF_frame=0, - LizardF_skippableFrame - LIZARDF_OBSOLETE_ENUM(skippableFrame = LizardF_skippableFrame) -} LizardF_frameType_t; - -#ifndef LIZARDF_DISABLE_OBSOLETE_ENUMS -typedef LizardF_blockSizeID_t blockSizeID_t; -typedef LizardF_blockMode_t blockMode_t; -typedef LizardF_frameType_t frameType_t; -typedef LizardF_contentChecksum_t contentChecksum_t; -#endif - -typedef struct { - LizardF_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ - LizardF_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ - LizardF_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ - LizardF_frameType_t frameType; /* LizardF_frame, skippableFrame ; 0 == default */ - unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */ - unsigned reserved[2]; /* must be zero for forward compatibility */ -} LizardF_frameInfo_t; - -typedef struct { - LizardF_frameInfo_t frameInfo; - int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */ - unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */ - unsigned reserved[4]; /* must be zero for forward compatibility */ -} LizardF_preferences_t; - - -/*-********************************* -* Simple compression function -***********************************/ -size_t LizardF_compressFrameBound(size_t srcSize, const LizardF_preferences_t* preferencesPtr); - -/*!LizardF_compressFrame() : - * Compress an entire srcBuffer into a valid Lizard frame, as defined by specification v1.5.1 - * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. - * You can get the minimum value of dstMaxSize by using LizardF_compressFrameBound() - * If this condition is not respected, LizardF_compressFrame() will fail (result is an errorCode) - * The LizardF_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. - * The result of the function is the number of bytes written into dstBuffer. - * The function outputs an error code if it fails (can be tested using LizardF_isError()) - */ -size_t LizardF_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_preferences_t* preferencesPtr); - - - -/*-*********************************** -* Advanced compression functions -*************************************/ -typedef struct LizardF_cctx_s* LizardF_compressionContext_t; /* must be aligned on 8-bytes */ - -typedef struct { - unsigned stableSrc; /* 1 == src content will remain available on future calls to LizardF_compress(); avoid saving src content within tmp buffer as future dictionary */ - unsigned reserved[3]; -} LizardF_compressOptions_t; - -/* Resource Management */ - -#define LIZARDF_VERSION 100 -LizardF_errorCode_t LizardF_createCompressionContext(LizardF_compressionContext_t* cctxPtr, unsigned version); -LizardF_errorCode_t LizardF_freeCompressionContext(LizardF_compressionContext_t cctx); -/* LizardF_createCompressionContext() : - * The first thing to do is to create a compressionContext object, which will be used in all compression operations. - * This is achieved using LizardF_createCompressionContext(), which takes as argument a version and an LizardF_preferences_t structure. - * The version provided MUST be LIZARDF_VERSION. It is intended to track potential version differences between different binaries. - * The function will provide a pointer to a fully allocated LizardF_compressionContext_t object. - * If the result LizardF_errorCode_t is not zero, there was an error during context creation. - * Object can release its memory using LizardF_freeCompressionContext(); - */ - - -/* Compression */ - -size_t LizardF_compressBegin(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LizardF_preferences_t* prefsPtr); -/* LizardF_compressBegin() : - * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes. - * The LizardF_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. - * The result of the function is the number of bytes written into dstBuffer for the header - * or an error code (can be tested using LizardF_isError()) - */ - -size_t LizardF_compressBound(size_t srcSize, const LizardF_preferences_t* prefsPtr); -/* LizardF_compressBound() : - * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. - * Different preferences can produce different results. - * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. - * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) - */ - -size_t LizardF_compressUpdate(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_compressOptions_t* cOptPtr); -/* LizardF_compressUpdate() - * LizardF_compressUpdate() can be called repetitively to compress as much data as necessary. - * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. - * You can get the minimum value of dstMaxSize by using LizardF_compressBound(). - * If this condition is not respected, LizardF_compress() will fail (result is an errorCode). - * LizardF_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs. - * The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. - * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. - * The function outputs an error code if it fails (can be tested using LizardF_isError()) - */ - -size_t LizardF_flush(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* cOptPtr); -/* LizardF_flush() - * Should you need to generate compressed data immediately, without waiting for the current block to be filled, - * you can call Lizard_flush(), which will immediately compress any remaining data buffered within cctx. - * Note that dstMaxSize must be large enough to ensure the operation will be successful. - * LizardF_compressOptions_t structure is optional : you can provide NULL as argument. - * The result of the function is the number of bytes written into dstBuffer - * (it can be zero, this means there was no data left within cctx) - * The function outputs an error code if it fails (can be tested using LizardF_isError()) - */ - -size_t LizardF_compressEnd(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* cOptPtr); -/* LizardF_compressEnd() - * When you want to properly finish the compressed frame, just call LizardF_compressEnd(). - * It will flush whatever data remained within compressionContext (like Lizard_flush()) - * but also properly finalize the frame, with an endMark and a checksum. - * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) - * The function outputs an error code if it fails (can be tested using LizardF_isError()) - * The LizardF_compressOptions_t structure is optional : you can provide NULL as argument. - * A successful call to LizardF_compressEnd() makes cctx available again for next compression task. - */ - - -/*-********************************* -* Decompression functions -***********************************/ - -typedef struct LizardF_dctx_s* LizardF_decompressionContext_t; /* must be aligned on 8-bytes */ - -typedef struct { - unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */ - unsigned reserved[3]; -} LizardF_decompressOptions_t; - - -/* Resource management */ - -/*!LizardF_createDecompressionContext() : - * Create an LizardF_decompressionContext_t object, which will be used to track all decompression operations. - * The version provided MUST be LIZARDF_VERSION. It is intended to track potential breaking differences between different versions. - * The function will provide a pointer to a fully allocated and initialized LizardF_decompressionContext_t object. - * The result is an errorCode, which can be tested using LizardF_isError(). - * dctx memory can be released using LizardF_freeDecompressionContext(); - * The result of LizardF_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. - * That is, it should be == 0 if decompression has been completed fully and correctly. - */ -LizardF_errorCode_t LizardF_createDecompressionContext(LizardF_decompressionContext_t* dctxPtr, unsigned version); -LizardF_errorCode_t LizardF_freeDecompressionContext(LizardF_decompressionContext_t dctx); - - -/*====== Decompression ======*/ - -/*!LizardF_getFrameInfo() : - * This function decodes frame header information (such as max blockSize, frame checksum, etc.). - * Its usage is optional. The objective is to extract frame header information, typically for allocation purposes. - * A header size is variable and can be from 7 to 15 bytes. It's also possible to input more bytes than that. - * The number of bytes read from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). - * (note that LizardF_getFrameInfo() can also be used anytime *after* starting decompression, in this case 0 input byte is enough) - * Frame header info is *copied into* an already allocated LizardF_frameInfo_t structure. - * The function result is an hint about how many srcSize bytes LizardF_decompress() expects for next call, - * or an error code which can be tested using LizardF_isError() - * (typically, when there is not enough src bytes to fully decode the frame header) - * Decompression is expected to resume from where it stopped (srcBuffer + *srcSizePtr) - */ -size_t LizardF_getFrameInfo(LizardF_decompressionContext_t dctx, - LizardF_frameInfo_t* frameInfoPtr, - const void* srcBuffer, size_t* srcSizePtr); - -/*!LizardF_decompress() : - * Call this function repetitively to regenerate data compressed within srcBuffer. - * The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr. - * - * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). - * - * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). - * If number of bytes read is < number of bytes provided, then decompression operation is not completed. - * It typically happens when dstBuffer is not large enough to contain all decoded data. - * LizardF_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr) - * The function will check this condition, and refuse to continue if it is not respected. - * - * `dstBuffer` is expected to be flushed between each call to the function, its content will be overwritten. - * `dst` arguments can be changed at will at each consecutive call to the function. - * - * The function result is an hint of how many `srcSize` bytes LizardF_decompress() expects for next call. - * Schematically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides some boost to performance, since it does skip intermediate buffers. - * This is just a hint though, it's always possible to provide any srcSize. - * When a frame is fully decoded, the function result will be 0 (no more data expected). - * If decompression failed, function result is an error code, which can be tested using LizardF_isError(). - * - * After a frame is fully decoded, dctx can be used again to decompress another frame. - */ -size_t LizardF_decompress(LizardF_decompressionContext_t dctx, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const LizardF_decompressOptions_t* dOptPtr); - - - -#if defined (__cplusplus) -} -#endif diff --git a/contrib/lizard/lib/lizard_frame_static.h b/contrib/lizard/lib/lizard_frame_static.h deleted file mode 100644 index da80756057b..00000000000 --- a/contrib/lizard/lib/lizard_frame_static.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - Lizard auto-framing library - Header File for static linking only - Copyright (C) 2011-2015, Yann Collet. - Copyright (C) 2016-2017, Przemyslaw Skibinski - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ - -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - -/* lizard_frame_static.h should be used solely in the context of static linking. - * It contains definitions which may still change overtime. - * Never use it in the context of DLL linking. - * */ - - -/************************************** -* Includes -**************************************/ -#include "lizard_frame.h" - - -/************************************** - * Error management - * ************************************/ -#define LIZARDF_LIST_ERRORS(ITEM) \ - ITEM(OK_NoError) ITEM(ERROR_GENERIC) \ - ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \ - ITEM(ERROR_compressionLevel_invalid) \ - ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \ - ITEM(ERROR_allocation_failed) \ - ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \ - ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \ - ITEM(ERROR_srcPtr_wrong) \ - ITEM(ERROR_decompressionFailed) \ - ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \ - ITEM(ERROR_maxCode) - -//#define LIZARDF_DISABLE_OLD_ENUMS -#ifndef LIZARDF_DISABLE_OLD_ENUMS -#define LIZARDF_GENERATE_ENUM(ENUM) LizardF_##ENUM, ENUM = LizardF_##ENUM, -#else -#define LIZARDF_GENERATE_ENUM(ENUM) LizardF_##ENUM, -#endif -typedef enum { LIZARDF_LIST_ERRORS(LIZARDF_GENERATE_ENUM) } LizardF_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */ - - -#if defined (__cplusplus) -} -#endif diff --git a/contrib/lizard/lib/lizard_parser_fast.h b/contrib/lizard/lib/lizard_parser_fast.h deleted file mode 100644 index f9e54b7f475..00000000000 --- a/contrib/lizard/lib/lizard_parser_fast.h +++ /dev/null @@ -1,196 +0,0 @@ -#define LIZARD_FAST_MIN_OFFSET 8 -#define LIZARD_FAST_LONGOFF_MM 0 /* not used with offsets > 1<<16 */ - -/************************************** -* Hash Functions -**************************************/ -static size_t Lizard_hashPosition(const void* p) -{ - if (MEM_64bits()) - return Lizard_hash5Ptr(p, LIZARD_HASHLOG_LZ4); - return Lizard_hash4Ptr(p, LIZARD_HASHLOG_LZ4); -} - -static void Lizard_putPositionOnHash(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase) -{ - hashTable[h] = (U32)(p-srcBase); -} - -static void Lizard_putPosition(const BYTE* p, U32* hashTable, const BYTE* srcBase) -{ - size_t const h = Lizard_hashPosition(p); - Lizard_putPositionOnHash(p, h, hashTable, srcBase); -} - -static U32 Lizard_getPositionOnHash(size_t h, U32* hashTable) -{ - return hashTable[h]; -} - -static U32 Lizard_getPosition(const BYTE* p, U32* hashTable) -{ - size_t const h = Lizard_hashPosition(p); - return Lizard_getPositionOnHash(h, hashTable); -} - - -static const U32 Lizard_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */ -static const U32 Lizard_minLength = (MFLIMIT+1); - - -FORCE_INLINE int Lizard_compress_fast( - Lizard_stream_t* const ctx, - const BYTE* ip, - const BYTE* const iend) -{ - const U32 acceleration = 1; - const BYTE* base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const BYTE* const dictBase = ctx->dictBase; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - const BYTE* anchor = ip; - - size_t forwardH, matchIndex; - const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip - base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; - - /* Init conditions */ - if ((U32)(iend-ip) > (U32)LIZARD_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */ - - if ((U32)(iend-ip) < Lizard_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ - - /* First Byte */ - Lizard_putPosition(ip, ctx->hashTable, base); - ip++; forwardH = Lizard_hashPosition(ip); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - size_t matchLength; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << Lizard_skipTrigger; - while (1) { - size_t const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> Lizard_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - matchIndex = Lizard_getPositionOnHash(h, ctx->hashTable); - forwardH = Lizard_hashPosition(forwardIp); - Lizard_putPositionOnHash(ip, h, ctx->hashTable, base); - - if ((matchIndex < lowLimit) || (matchIndex >= (U32)(ip - base)) || (base + matchIndex + maxDistance < ip)) continue; - - if (matchIndex >= dictLimit) { - match = base + matchIndex; -#if LIZARD_FAST_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) -#endif - if (MEM_read32(match) == MEM_read32(ip)) - { - int back = 0; - matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); - - while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; - matchLength -= back; -#if LIZARD_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) -#endif - { - ip += back; - match += back; - break; - } - } - } else { - match = dictBase + matchIndex; -#if LIZARD_FAST_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) -#endif - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance; - int back = 0; - matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); - - while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--; - matchLength -= back; - match = base + matchIndex + back; -#if LIZARD_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) -#endif - { - ip += back; - break; - } - } - } - } // while (1) - } - -_next_match: - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error; - - /* Test end of chunk */ - if (ip > mflimit) break; - - /* Fill table */ - Lizard_putPosition(ip-2, ctx->hashTable, base); - - /* Test next position */ - matchIndex = Lizard_getPosition(ip, ctx->hashTable); - Lizard_putPosition(ip, ctx->hashTable, base); - if ((matchIndex >= lowLimit) && (matchIndex < (U32)(ip - base)) && (base + matchIndex + maxDistance >= ip)) - { - if (matchIndex >= dictLimit) { - match = base + matchIndex; -#if LIZARD_FAST_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) -#endif - if (MEM_read32(match) == MEM_read32(ip)) - { - matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); -#if LIZARD_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) -#endif - goto _next_match; - } - } else { - match = dictBase + matchIndex; -#if LIZARD_FAST_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) -#endif - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); - match = base + matchIndex; -#if LIZARD_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) -#endif - goto _next_match; - } - } - } - - /* Prepare next loop */ - forwardH = Lizard_hashPosition(++ip); - } - -_last_literals: - /* Encode Last Literals */ - ip = iend; - if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; - - /* End */ - return 1; -_output_error: - return 0; -} diff --git a/contrib/lizard/lib/lizard_parser_fastbig.h b/contrib/lizard/lib/lizard_parser_fastbig.h deleted file mode 100644 index 30c464fcb90..00000000000 --- a/contrib/lizard/lib/lizard_parser_fastbig.h +++ /dev/null @@ -1,175 +0,0 @@ -#define LIZARD_FASTBIG_LONGOFF_MM MM_LONGOFF - -/************************************** -* Hash Functions -**************************************/ -static size_t Lizard_hashPositionHLog(const void* p, int hashLog) -{ - if (MEM_64bits()) - return Lizard_hash5Ptr(p, hashLog); - return Lizard_hash4Ptr(p, hashLog); -} - -static void Lizard_putPositionOnHashHLog(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase) -{ - hashTable[h] = (U32)(p-srcBase); -} - -static void Lizard_putPositionHLog(const BYTE* p, U32* hashTable, const BYTE* srcBase, int hashLog) -{ - size_t const h = Lizard_hashPositionHLog(p, hashLog); - Lizard_putPositionOnHashHLog(p, h, hashTable, srcBase); -} - -static U32 Lizard_getPositionOnHashHLog(size_t h, U32* hashTable) -{ - return hashTable[h]; -} - -static U32 Lizard_getPositionHLog(const BYTE* p, U32* hashTable, int hashLog) -{ - size_t const h = Lizard_hashPositionHLog(p, hashLog); - return Lizard_getPositionOnHashHLog(h, hashTable); -} - -FORCE_INLINE int Lizard_compress_fastBig( - Lizard_stream_t* const ctx, - const BYTE* ip, - const BYTE* const iend) -{ - const U32 acceleration = 1; - const BYTE* base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const BYTE* const dictBase = ctx->dictBase; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - const BYTE* anchor = ip; - - size_t forwardH, matchIndex; - const int hashLog = ctx->params.hashLog; - const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip - base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; - - /* Init conditions */ - if ((U32)(iend-ip) > (U32)LIZARD_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */ - - if ((U32)(iend-ip) < Lizard_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ - - /* First Byte */ - Lizard_putPositionHLog(ip, ctx->hashTable, base, hashLog); - ip++; forwardH = Lizard_hashPositionHLog(ip, hashLog); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - size_t matchLength; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << Lizard_skipTrigger; - while (1) { - size_t const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> Lizard_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - matchIndex = Lizard_getPositionOnHashHLog(h, ctx->hashTable); - forwardH = Lizard_hashPositionHLog(forwardIp, hashLog); - Lizard_putPositionOnHashHLog(ip, h, ctx->hashTable, base); - - if ((matchIndex < lowLimit) || (matchIndex >= (U32)(ip - base)) || (base + matchIndex + maxDistance < ip)) continue; - - if (matchIndex >= dictLimit) { - match = base + matchIndex; - if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) - if (MEM_read32(match) == MEM_read32(ip)) - { - int back = 0; - matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); - - while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; - matchLength -= back; - if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - { - ip += back; - match += back; - break; - } - } - } else { - match = dictBase + matchIndex; - if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance; - int back = 0; - matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); - - while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--; - matchLength -= back; - match = base + matchIndex + back; - if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - { - ip += back; - break; - } - } - } - } // while (1) - } - -_next_match: - if (Lizard_encodeSequence_LIZv1(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error; - - /* Test end of chunk */ - if (ip > mflimit) break; - - /* Fill table */ - Lizard_putPositionHLog(ip-2, ctx->hashTable, base, hashLog); - - /* Test next position */ - matchIndex = Lizard_getPositionHLog(ip, ctx->hashTable, hashLog); - Lizard_putPositionHLog(ip, ctx->hashTable, base, hashLog); - if ((matchIndex >= lowLimit) && (matchIndex < (U32)(ip - base)) && (base + matchIndex + maxDistance >= ip)) - { - if (matchIndex >= dictLimit) { - match = base + matchIndex; - if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) - if (MEM_read32(match) == MEM_read32(ip)) - { - matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); - if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - goto _next_match; - } - } else { - match = dictBase + matchIndex; - if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); - match = base + matchIndex; - if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - goto _next_match; - } - } - } - - /* Prepare next loop */ - forwardH = Lizard_hashPositionHLog(++ip, hashLog); - } - -_last_literals: - /* Encode Last Literals */ - ip = iend; - if (Lizard_encodeLastLiterals_LIZv1(ctx, &ip, &anchor)) goto _output_error; - - /* End */ - return 1; -_output_error: - return 0; -} diff --git a/contrib/lizard/lib/lizard_parser_fastsmall.h b/contrib/lizard/lib/lizard_parser_fastsmall.h deleted file mode 100644 index 550edc5b201..00000000000 --- a/contrib/lizard/lib/lizard_parser_fastsmall.h +++ /dev/null @@ -1,189 +0,0 @@ -/************************************** -* Hash Functions -**************************************/ -static size_t Lizard_hashPositionSmall(const void* p) -{ - if (MEM_64bits()) - return Lizard_hash5Ptr(p, LIZARD_HASHLOG_LZ4SM); - return Lizard_hash4Ptr(p, LIZARD_HASHLOG_LZ4SM); -} - -static void Lizard_putPositionOnHashSmall(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase) -{ - hashTable[h] = (U32)(p-srcBase); -} - -static void Lizard_putPositionSmall(const BYTE* p, U32* hashTable, const BYTE* srcBase) -{ - size_t const h = Lizard_hashPositionSmall(p); - Lizard_putPositionOnHashSmall(p, h, hashTable, srcBase); -} - -static U32 Lizard_getPositionOnHashSmall(size_t h, U32* hashTable) -{ - return hashTable[h]; -} - -static U32 Lizard_getPositionSmall(const BYTE* p, U32* hashTable) -{ - size_t const h = Lizard_hashPositionSmall(p); - return Lizard_getPositionOnHashSmall(h, hashTable); -} - - -FORCE_INLINE int Lizard_compress_fastSmall( - Lizard_stream_t* const ctx, - const BYTE* ip, - const BYTE* const iend) -{ - const U32 acceleration = 1; - const BYTE* base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const BYTE* const dictBase = ctx->dictBase; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - const BYTE* anchor = ip; - - size_t forwardH, matchIndex; - const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip - base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance; - - /* Init conditions */ - if ((U32)(iend-ip) > (U32)LIZARD_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */ - - if ((U32)(iend-ip) < Lizard_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ - - /* First Byte */ - Lizard_putPositionSmall(ip, ctx->hashTable, base); - ip++; forwardH = Lizard_hashPositionSmall(ip); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - size_t matchLength; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << Lizard_skipTrigger; - while (1) { - size_t const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> Lizard_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - matchIndex = Lizard_getPositionOnHashSmall(h, ctx->hashTable); - forwardH = Lizard_hashPositionSmall(forwardIp); - Lizard_putPositionOnHashSmall(ip, h, ctx->hashTable, base); - - if ((matchIndex < lowLimit) || (matchIndex >= (U32)(ip - base)) || (base + matchIndex + maxDistance < ip)) continue; - - if (matchIndex >= dictLimit) { - match = base + matchIndex; -#if LIZARD_FAST_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) -#endif - if (MEM_read32(match) == MEM_read32(ip)) - { - int back = 0; - matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); - - while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; - matchLength -= back; -#if LIZARD_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) -#endif - { - ip += back; - match += back; - break; - } - } - } else { - match = dictBase + matchIndex; -#if LIZARD_FAST_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) -#endif - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance; - int back = 0; - matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); - - while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--; - matchLength -= back; - match = base + matchIndex + back; -#if LIZARD_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) -#endif - { - ip += back; - break; - } - } - } - } // while (1) - } - -_next_match: - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error; - - /* Test end of chunk */ - if (ip > mflimit) break; - - /* Fill table */ - Lizard_putPositionSmall(ip-2, ctx->hashTable, base); - - /* Test next position */ - matchIndex = Lizard_getPositionSmall(ip, ctx->hashTable); - Lizard_putPositionSmall(ip, ctx->hashTable, base); - if ((matchIndex >= lowLimit) && (matchIndex < (U32)(ip - base)) && (base + matchIndex + maxDistance >= ip)) - { - if (matchIndex >= dictLimit) { - match = base + matchIndex; -#if LIZARD_FAST_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET) -#endif - if (MEM_read32(match) == MEM_read32(ip)) - { - matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit); -#if LIZARD_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) -#endif - goto _next_match; - } - } else { - match = dictBase + matchIndex; -#if LIZARD_FAST_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET) -#endif - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr); - match = base + matchIndex; -#if LIZARD_FAST_LONGOFF_MM > 0 - if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) -#endif - goto _next_match; - } - } - } - - /* Prepare next loop */ - forwardH = Lizard_hashPositionSmall(++ip); - } - -_last_literals: - /* Encode Last Literals */ - ip = iend; - if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; - - /* End */ - return 1; -_output_error: - return 0; -} diff --git a/contrib/lizard/lib/lizard_parser_hashchain.h b/contrib/lizard/lib/lizard_parser_hashchain.h deleted file mode 100644 index 64b9d76c451..00000000000 --- a/contrib/lizard/lib/lizard_parser_hashchain.h +++ /dev/null @@ -1,369 +0,0 @@ -#define LIZARD_HC_MIN_OFFSET 8 -#define LIZARD_HC_LONGOFF_MM 0 /* not used with offsets > 1<<16 */ -#define OPTIMAL_ML (int)((ML_MASK_LZ4-1)+MINMATCH) -#define GET_MINMATCH(offset) (MINMATCH) - -#if 1 - #define LIZARD_HC_HASH_FUNCTION(ip, hashLog) Lizard_hashPtr(ip, hashLog, ctx->params.searchLength) -#else - #define LIZARD_HC_HASH_FUNCTION(ip, hashLog) Lizard_hash5Ptr(ip, hashLog) -#endif - -/* Update chains up to ip (excluded) */ -FORCE_INLINE void Lizard_Insert (Lizard_stream_t* ctx, const BYTE* ip) -{ - U32* const chainTable = ctx->chainTable; - U32* const hashTable = ctx->hashTable; -#if MINMATCH == 3 - U32* HashTable3 = ctx->hashTable3; -#endif - const BYTE* const base = ctx->base; - U32 const target = (U32)(ip - base); - U32 idx = ctx->nextToUpdate; - const int hashLog = ctx->params.hashLog; - const U32 contentMask = (1 << ctx->params.contentLog) - 1; - const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - - while (idx < target) { - size_t const h = Lizard_hashPtr(base+idx, hashLog, ctx->params.searchLength); - size_t delta = idx - hashTable[h]; - if (delta>maxDistance) delta = maxDistance; - DELTANEXT(idx) = (U32)delta; - if ((hashTable[h] >= idx) || (idx >= hashTable[h] + LIZARD_HC_MIN_OFFSET)) - hashTable[h] = idx; -#if MINMATCH == 3 - HashTable3[Lizard_hash3Ptr(base+idx, ctx->params.hashLog3)] = idx; -#endif - idx++; - } - - ctx->nextToUpdate = target; -} - - - -FORCE_INLINE int Lizard_InsertAndFindBestMatch (Lizard_stream_t* ctx, /* Index table will be updated */ - const BYTE* ip, const BYTE* const iLimit, - const BYTE** matchpos) -{ - U32* const chainTable = ctx->chainTable; - U32* const HashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - const BYTE* const dictBase = ctx->dictBase; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - U32 matchIndex, delta; - const BYTE* match; - int nbAttempts=ctx->params.searchNum; - size_t ml=0; - const int hashLog = ctx->params.hashLog; - const U32 contentMask = (1 << ctx->params.contentLog) - 1; - const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - const U32 current = (U32)(ip - base); - const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; - - /* HC4 match finder */ - Lizard_Insert(ctx, ip); - matchIndex = HashTable[LIZARD_HC_HASH_FUNCTION(ip, hashLog)]; - - while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { - nbAttempts--; - if (matchIndex >= dictLimit) { - match = base + matchIndex; -#if LIZARD_HC_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZARD_HC_MIN_OFFSET) -#endif - if (*(match+ml) == *(ip+ml) - && (MEM_read32(match) == MEM_read32(ip))) - { - size_t const mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; -#if LIZARD_HC_LONGOFF_MM > 0 - if ((mlt >= LIZARD_HC_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) -#endif - if (mlt > ml) { ml = mlt; *matchpos = match; } - } - } else { - match = dictBase + matchIndex; -#if LIZARD_HC_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZARD_HC_MIN_OFFSET) -#endif - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - size_t mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; -#if LIZARD_HC_LONGOFF_MM > 0 - if ((mlt >= LIZARD_HC_LONGOFF_MM) || ((U32)(ip - (base + matchIndex)) < LIZARD_MAX_16BIT_OFFSET)) -#endif - if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ - } - } - delta = DELTANEXT(matchIndex); - if (delta > matchIndex) break; - matchIndex -= delta; - } - - return (int)ml; -} - - -FORCE_INLINE int Lizard_InsertAndGetWiderMatch ( - Lizard_stream_t* ctx, - const BYTE* const ip, - const BYTE* const iLowLimit, - const BYTE* const iHighLimit, - int longest, - const BYTE** matchpos, - const BYTE** startpos) -{ - U32* const chainTable = ctx->chainTable; - U32* const HashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const BYTE* const dictBase = ctx->dictBase; - const BYTE* const dictEnd = dictBase + dictLimit; - U32 matchIndex, delta; - int nbAttempts = ctx->params.searchNum; - int LLdelta = (int)(ip-iLowLimit); - const int hashLog = ctx->params.hashLog; - const U32 contentMask = (1 << ctx->params.contentLog) - 1; - const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - const U32 current = (U32)(ip - base); - const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; - - /* First Match */ - Lizard_Insert(ctx, ip); - matchIndex = HashTable[LIZARD_HC_HASH_FUNCTION(ip, hashLog)]; - - while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { - nbAttempts--; - if (matchIndex >= dictLimit) { - const BYTE* match = base + matchIndex; -#if LIZARD_HC_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZARD_HC_MIN_OFFSET) -#endif - if (*(iLowLimit + longest) == *(match - LLdelta + longest)) { - if (MEM_read32(match) == MEM_read32(ip)) { - int mlt = MINMATCH + Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit); - int back = 0; - while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; - mlt -= back; - -#if LIZARD_HC_LONGOFF_MM > 0 - if ((mlt >= LIZARD_HC_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) -#endif - if (mlt > longest) { - longest = (int)mlt; - *matchpos = match+back; - *startpos = ip+back; - } - } - } - } else { - const BYTE* match = dictBase + matchIndex; -#if LIZARD_HC_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZARD_HC_MIN_OFFSET) -#endif - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - int back=0; - size_t mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; - while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; - mlt -= back; -#if LIZARD_HC_LONGOFF_MM > 0 - if ((mlt >= LIZARD_HC_LONGOFF_MM) || ((U32)(ip - (base + matchIndex)) < LIZARD_MAX_16BIT_OFFSET)) -#endif - if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } - } - } - delta = DELTANEXT(matchIndex); - if (delta > matchIndex) break; - matchIndex -= delta; - } - - return longest; -} - - -FORCE_INLINE int Lizard_compress_hashChain ( - Lizard_stream_t* const ctx, - const BYTE* ip, - const BYTE* const iend) -{ - const BYTE* anchor = ip; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); - - int ml, ml2, ml3, ml0; - const BYTE* ref = NULL; - const BYTE* start2 = NULL; - const BYTE* ref2 = NULL; - const BYTE* start3 = NULL; - const BYTE* ref3 = NULL; - const BYTE* start0; - const BYTE* ref0; - - /* init */ - ip++; - - /* Main Loop */ - while (ip < mflimit) { - ml = Lizard_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref)); - if (!ml) { ip++; continue; } - - /* saved, in case we would skip too much */ - start0 = ip; - ref0 = ref; - ml0 = ml; - -_Search2: - if (ip+ml < mflimit) - ml2 = Lizard_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2); - else ml2 = ml; - - if (ml2 == ml) { /* No better match */ - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; - continue; - } - - if (start0 < ip) { - if (start2 < ip + ml0) { /* empirical */ - ip = start0; - ref = ref0; - ml = ml0; - } - } - - /* Here, start0==ip */ - if ((start2 - ip) < 3) { /* First Match too small : removed */ - ml = ml2; - ip = start2; - ref =ref2; - goto _Search2; - } - -_Search3: - /* - * Currently we have : - * ml2 > ml1, and - * ip1+3 <= ip2 (usually < ip1+ml1) - */ - if ((start2 - ip) < OPTIMAL_ML) { - int correction; - int new_ml = ml; - if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; - if (ip+new_ml > start2 + ml2 - GET_MINMATCH((U32)(start2 - ref2))) { - new_ml = (int)(start2 - ip) + ml2 - GET_MINMATCH((U32)(start2 - ref2)); - if (new_ml < GET_MINMATCH((U32)(ip - ref))) { // match2 doesn't fit - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; - continue; - } - } - correction = new_ml - (int)(start2 - ip); - if (correction > 0) { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } - /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ - - if (start2 + ml2 < mflimit) - ml3 = Lizard_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3); - else ml3 = ml2; - - if (ml3 == ml2) { /* No better match : 2 sequences to encode */ - /* ip & ref are known; Now for ml */ - if (start2 < ip+ml) ml = (int)(start2 - ip); - /* Now, encode 2 sequences */ - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; - ip = start2; - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml2, ref2)) return 0; - continue; - } - - if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */ - if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ - if (start2 < ip+ml) { - int correction = (int)(ip+ml - start2); - start2 += correction; - ref2 += correction; - ml2 -= correction; - if (ml2 < GET_MINMATCH((U32)(start2 - ref2))) { - start2 = start3; - ref2 = ref3; - ml2 = ml3; - } - } - - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; - ip = start3; - ref = ref3; - ml = ml3; - - start0 = start2; - ref0 = ref2; - ml0 = ml2; - goto _Search2; - } - - start2 = start3; - ref2 = ref3; - ml2 = ml3; - goto _Search3; - } - - /* - * OK, now we have 3 ascending matches; let's write at least the first one - * ip & ref are known; Now for ml - */ - if (start2 < ip+ml) { - if ((start2 - ip) < (int)ML_MASK_LZ4) { - int correction; - if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; - if (ip + ml > start2 + ml2 - GET_MINMATCH((U32)(start2 - ref2))) { - ml = (int)(start2 - ip) + ml2 - GET_MINMATCH((U32)(start2 - ref2)); - if (ml < GET_MINMATCH((U32)(ip - ref))) { // match2 doesn't fit, remove it - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; - ip = start3; - ref = ref3; - ml = ml3; - - start0 = start2; - ref0 = ref2; - ml0 = ml2; - goto _Search2; - } - } - correction = ml - (int)(start2 - ip); - if (correction > 0) { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } else { - ml = (int)(start2 - ip); - } - } - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; - - ip = start2; - ref = ref2; - ml = ml2; - - start2 = start3; - ref2 = ref3; - ml2 = ml3; - - goto _Search3; - } - - /* Encode Last Literals */ - ip = iend; - if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; - - /* End */ - return 1; -_output_error: - return 0; -} diff --git a/contrib/lizard/lib/lizard_parser_lowestprice.h b/contrib/lizard/lib/lizard_parser_lowestprice.h deleted file mode 100644 index f8b95dad4ea..00000000000 --- a/contrib/lizard/lib/lizard_parser_lowestprice.h +++ /dev/null @@ -1,376 +0,0 @@ -#define LIZARD_LOWESTPRICE_MIN_OFFSET 8 - - -FORCE_INLINE size_t Lizard_more_profitable(Lizard_stream_t* const ctx, const BYTE *best_ip, size_t best_off, size_t best_common, const BYTE *ip, size_t off, size_t common, size_t literals, int last_off) -{ - size_t sum; - - if (literals > 0) - sum = MAX(common + literals, best_common); - else - sum = MAX(common, best_common - literals); - - if ((int)off == last_off) off = 0; // rep code - if ((int)best_off == last_off) best_off = 0; - - return Lizard_get_price_LIZv1(ctx, last_off, ip, ctx->off24pos, sum - common, (U32)off, common) <= Lizard_get_price_LIZv1(ctx, last_off, best_ip, ctx->off24pos, sum - best_common, (U32)best_off, best_common); -} - - -FORCE_INLINE size_t Lizard_better_price(Lizard_stream_t* const ctx, const BYTE *best_ip, size_t best_off, size_t best_common, const BYTE *ip, size_t off, size_t common, int last_off) -{ - if ((int)off == last_off) off = 0; // rep code - if ((int)best_off == last_off) best_off = 0; - - return Lizard_get_price_LIZv1(ctx, last_off, ip, ctx->off24pos, 0, (U32)off, common) < Lizard_get_price_LIZv1(ctx, last_off, best_ip, ctx->off24pos, common - best_common, (U32)best_off, best_common); -} - - -FORCE_INLINE int Lizard_FindMatchLowestPrice (Lizard_stream_t* ctx, /* Index table will be updated */ - const BYTE* ip, const BYTE* const iLimit, - const BYTE** matchpos) -{ - U32* const chainTable = ctx->chainTable; - U32* const HashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - const BYTE* const dictBase = ctx->dictBase; - const intptr_t dictLimit = ctx->dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1; - const intptr_t current = (ip - base); - const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance; - const BYTE* const lowPrefixPtr = base + dictLimit; - const U32 contentMask = (1 << ctx->params.contentLog) - 1; - const size_t minMatchLongOff = ctx->params.minMatchLongOff; - intptr_t matchIndex; - const BYTE* match, *matchDict; - int nbAttempts=ctx->params.searchNum; - size_t ml=0, mlt; - - matchIndex = HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; - - if (ctx->last_off >= LIZARD_LOWESTPRICE_MIN_OFFSET) { - intptr_t matchIndexLO = (ip - ctx->last_off) - base; - if (matchIndexLO >= lowLimit) { - if (matchIndexLO >= dictLimit) { - match = base + matchIndexLO; - mlt = Lizard_count(ip, match, iLimit);// + MINMATCH; - // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) - if (mlt > REPMINMATCH) { - *matchpos = match; - return (int)mlt; - } - } else { - match = dictBase + matchIndexLO; - if ((U32)((dictLimit-1) - matchIndexLO) >= 3) { /* intentional overflow */ - mlt = Lizard_count_2segments(ip, match, iLimit, dictEnd, lowPrefixPtr); - // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) - if (mlt > REPMINMATCH) { - *matchpos = base + matchIndexLO; /* virtual matchpos */ - return (int)mlt; - } - } - } - } - } - - -#if MINMATCH == 3 - { - U32 matchIndex3 = ctx->hashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; - if (matchIndex3 < current && matchIndex3 >= lowLimit) - { - size_t offset = (size_t)current - matchIndex3; - if (offset < LIZARD_MAX_8BIT_OFFSET) - { - match = ip - offset; - if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) - { - ml = 3;//Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - *matchpos = match; - } - } - } - } -#endif - while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { - nbAttempts--; - match = base + matchIndex; - if ((U32)(ip - match) >= LIZARD_LOWESTPRICE_MIN_OFFSET) { - if (matchIndex >= dictLimit) { - if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) { - mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - if (!ml || (mlt > ml && Lizard_better_price(ctx, ip, (ip - *matchpos), ml, ip, (ip - match), mlt, ctx->last_off))) - { ml = mlt; *matchpos = match; } - } - } else { - matchDict = dictBase + matchIndex; - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(matchDict) == MEM_read32(ip)) { - mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - if (!ml || (mlt > ml && Lizard_better_price(ctx, ip, (ip - *matchpos), ml, ip, (U32)(ip - match), mlt, ctx->last_off))) - { ml = mlt; *matchpos = match; } /* virtual matchpos */ - } - } - } - matchIndex -= chainTable[matchIndex & contentMask]; - } - - return (int)ml; -} - - -FORCE_INLINE size_t Lizard_GetWiderMatch ( - Lizard_stream_t* ctx, - const BYTE* const ip, - const BYTE* const iLowLimit, - const BYTE* const iHighLimit, - size_t longest, - const BYTE** matchpos, - const BYTE** startpos) -{ - U32* const chainTable = ctx->chainTable; - U32* const HashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - const BYTE* const dictBase = ctx->dictBase; - const intptr_t dictLimit = ctx->dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1; - const intptr_t current = (ip - base); - const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance; - const BYTE* const lowPrefixPtr = base + dictLimit; - const U32 contentMask = (1 << ctx->params.contentLog) - 1; - const BYTE* match, *matchDict; - const size_t minMatchLongOff = ctx->params.minMatchLongOff; - intptr_t matchIndex; - int nbAttempts = ctx->params.searchNum; - size_t mlt; - - /* First Match */ - matchIndex = HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; - - if (ctx->last_off >= LIZARD_LOWESTPRICE_MIN_OFFSET) { - intptr_t matchIndexLO = (ip - ctx->last_off) - base; - if (matchIndexLO >= lowLimit) { - if (matchIndexLO >= dictLimit) { - match = base + matchIndexLO; - if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) { - int back = 0; - mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; - while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; - mlt -= back; - - if (mlt > longest) - if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) { - *matchpos = match+back; - *startpos = ip+back; - longest = mlt; - } - } - } else { - match = dictBase + matchIndexLO; - if ((U32)((dictLimit-1) - matchIndexLO) >= 3) /* intentional overflow */ - if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) { - int back=0; - mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; - while ((ip+back > iLowLimit) && (matchIndexLO+back > lowLimit) && (ip[back-1] == match[back-1])) back--; - mlt -= back; - - if (mlt > longest) - if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) { - *matchpos = base + matchIndexLO + back; /* virtual matchpos */ - *startpos = ip+back; - longest = mlt; - } - } - } - } - } - -#if MINMATCH == 3 - { - U32 matchIndex3 = ctx->hashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; - if (matchIndex3 < current && matchIndex3 >= lowLimit) { - size_t offset = (size_t)current - matchIndex3; - if (offset < LIZARD_MAX_8BIT_OFFSET) { - match = ip - offset; - if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) { - mlt = Lizard_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; - - int back = 0; - while ((ip + back > iLowLimit) && (match + back > lowPrefixPtr) && (ip[back - 1] == match[back - 1])) back--; - mlt -= back; - - if (!longest || (mlt > longest && Lizard_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (ip - match), mlt, ctx->last_off))) { - *matchpos = match + back; - *startpos = ip + back; - longest = mlt; - } - } - } - } - } -#endif - - while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { - nbAttempts--; - match = base + matchIndex; - if ((U32)(ip - match) >= LIZARD_LOWESTPRICE_MIN_OFFSET) { - if (matchIndex >= dictLimit) { - if (MEM_read32(match) == MEM_read32(ip)) { - int back = 0; - mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; - while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; - mlt -= back; - - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - if (!longest || (mlt > longest && Lizard_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (ip - match), mlt, ctx->last_off))) - { longest = mlt; *startpos = ip+back; *matchpos = match+back; } - } - } else { - matchDict = dictBase + matchIndex; - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(matchDict) == MEM_read32(ip)) { - int back=0; - mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; - while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchDict[back-1])) back--; - mlt -= back; - - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - if (!longest || (mlt > longest && Lizard_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (U32)(ip - match), mlt, ctx->last_off))) - { longest = mlt; *startpos = ip+back; *matchpos = match+back; } /* virtual matchpos */ - } - } - } - matchIndex -= chainTable[matchIndex & contentMask]; - } - - return longest; -} - - - - -FORCE_INLINE int Lizard_compress_lowestPrice( - Lizard_stream_t* const ctx, - const BYTE* ip, - const BYTE* const iend) -{ - const BYTE* anchor = ip; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); - - size_t ml, ml2, ml0; - const BYTE* ref=NULL; - const BYTE* start2=NULL; - const BYTE* ref2=NULL; - const BYTE* start0; - const BYTE* ref0; - const BYTE* lowPrefixPtr = ctx->base + ctx->dictLimit; - const size_t minMatchLongOff = ctx->params.minMatchLongOff; - const size_t sufficient_len = ctx->params.sufficientLength; - - /* Main Loop */ - while (ip < mflimit) - { - Lizard_Insert(ctx, ip); - ml = Lizard_FindMatchLowestPrice (ctx, ip, matchlimit, (&ref)); - if (!ml) { ip++; continue; } - - { - int back = 0; - while ((ip + back > anchor) && (ref + back > lowPrefixPtr) && (ip[back - 1] == ref[back - 1])) back--; - ml -= back; - ip += back; - ref += back; - } - - /* saved, in case we would skip too much */ - start0 = ip; - ref0 = ref; - ml0 = ml; - // goto _Encode; - -_Search: - if (ip+ml >= mflimit) { goto _Encode; } - if (ml >= sufficient_len) { goto _Encode; } - - Lizard_Insert(ctx, ip); - ml2 = (int)Lizard_GetWiderMatch(ctx, ip + ml - 2, anchor, matchlimit, 0, &ref2, &start2); - if (!ml2) goto _Encode; - - { - U64 price, best_price; - int off0=0, off1=0; - const BYTE *pos, *best_pos; - - // find the lowest price for encoding ml bytes - best_pos = ip; - best_price = LIZARD_MAX_PRICE; - off0 = (int)(ip - ref); - off1 = (int)(start2 - ref2); - - for (pos = ip + ml; pos >= start2; pos--) - { - int common0 = (int)(pos - ip); - if (common0 >= MINMATCH) { - price = (int)Lizard_get_price_LIZv1(ctx, ctx->last_off, ip, ctx->off24pos, ip - anchor, (off0 == ctx->last_off) ? 0 : off0, common0); - - { - int common1 = (int)(start2 + ml2 - pos); - if (common1 >= MINMATCH) - price += Lizard_get_price_LIZv1(ctx, ctx->last_off, pos, ctx->off24pos, 0, (off1 == off0) ? 0 : (off1), common1); - else - price += Lizard_get_price_LIZv1(ctx, ctx->last_off, pos, ctx->off24pos, common1, 0, 0); - } - - if (price < best_price) { - best_price = price; - best_pos = pos; - } - } else { - price = Lizard_get_price_LIZv1(ctx, ctx->last_off, ip, ctx->off24pos, start2 - anchor, (off1 == ctx->last_off) ? 0 : off1, ml2); - - if (price < best_price) - best_pos = pos; - break; - } - } - ml = (int)(best_pos - ip); - } - - - if ((ml < MINMATCH) || ((ml < minMatchLongOff) && ((U32)(ip-ref) >= LIZARD_MAX_16BIT_OFFSET))) - { - ip = start2; - ref = ref2; - ml = ml2; - goto _Search; - } - -_Encode: - if (start0 < ip) - { - if (Lizard_more_profitable(ctx, ip, (ip - ref), ml, start0, (start0 - ref0), ml0, (ref0 - ref), ctx->last_off)) - { - ip = start0; - ref = ref0; - ml = ml0; - } - } - - if (Lizard_encodeSequence_LIZv1(ctx, &ip, &anchor, ml, ((ip - ref == ctx->last_off) ? ip : ref))) return 0; - } - - /* Encode Last Literals */ - ip = iend; - if (Lizard_encodeLastLiterals_LIZv1(ctx, &ip, &anchor)) goto _output_error; - - /* End */ - return 1; -_output_error: - return 0; -} - diff --git a/contrib/lizard/lib/lizard_parser_nochain.h b/contrib/lizard/lib/lizard_parser_nochain.h deleted file mode 100644 index e8abb586ff4..00000000000 --- a/contrib/lizard/lib/lizard_parser_nochain.h +++ /dev/null @@ -1,318 +0,0 @@ -#define OPTIMAL_ML (int)((ML_MASK_LZ4-1)+MINMATCH) - -//#define LIZARD_NOCHAIN_HASH_FUNCTION(ip, hashLog) Lizard_hashPtr(ip, hashLog, ctx->params.searchLength) -#define LIZARD_NOCHAIN_HASH_FUNCTION(ip, hashLog) Lizard_hash5Ptr(ip, hashLog) -#define LIZARD_NOCHAIN_MIN_OFFSET 8 - -/* Update chains up to ip (excluded) */ -FORCE_INLINE void Lizard_InsertNoChain (Lizard_stream_t* ctx, const BYTE* ip) -{ - U32* const hashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - U32 const target = (U32)(ip - base); - U32 idx = ctx->nextToUpdate; - const int hashLog = ctx->params.hashLog; - - while (idx < target) { - size_t const h = LIZARD_NOCHAIN_HASH_FUNCTION(base+idx, hashLog); - if ((hashTable[h] >= idx) || (idx >= hashTable[h] + LIZARD_NOCHAIN_MIN_OFFSET)) - hashTable[h] = idx; - idx++; - } - - ctx->nextToUpdate = target; -} - - -FORCE_INLINE int Lizard_InsertAndFindBestMatchNoChain (Lizard_stream_t* ctx, /* Index table will be updated */ - const BYTE* ip, const BYTE* const iLimit, - const BYTE** matchpos) -{ - U32* const HashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - const BYTE* const dictBase = ctx->dictBase; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - U32 matchIndex; - const BYTE* match; - size_t ml=0; - const int hashLog = ctx->params.hashLog; - const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - const U32 current = (U32)(ip - base); - const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; - - /* HC4 match finder */ - Lizard_InsertNoChain(ctx, ip); - matchIndex = HashTable[LIZARD_NOCHAIN_HASH_FUNCTION(ip, hashLog)]; - - if ((matchIndex < current) && (matchIndex >= lowLimit)) { - if (matchIndex >= dictLimit) { - match = base + matchIndex; -#if LIZARD_NOCHAIN_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZARD_NOCHAIN_MIN_OFFSET) -#endif - if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) - { - size_t const mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - if (mlt > ml) { ml = mlt; *matchpos = match; } - } - } else { - match = dictBase + matchIndex; -#if LIZARD_NOCHAIN_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZARD_NOCHAIN_MIN_OFFSET) -#endif - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - size_t mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; - if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ - } - } - } - - return (int)ml; -} - - -FORCE_INLINE int Lizard_InsertAndGetWiderMatchNoChain ( - Lizard_stream_t* ctx, - const BYTE* const ip, - const BYTE* const iLowLimit, - const BYTE* const iHighLimit, - int longest, - const BYTE** matchpos, - const BYTE** startpos) -{ - U32* const HashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const BYTE* const dictBase = ctx->dictBase; - const BYTE* const dictEnd = dictBase + dictLimit; - U32 matchIndex; - int LLdelta = (int)(ip-iLowLimit); - const int hashLog = ctx->params.hashLog; - const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - const U32 current = (U32)(ip - base); - const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; - - /* First Match */ - Lizard_InsertNoChain(ctx, ip); - matchIndex = HashTable[LIZARD_NOCHAIN_HASH_FUNCTION(ip, hashLog)]; - - if ((matchIndex < current) && (matchIndex >= lowLimit)) { - if (matchIndex >= dictLimit) { - const BYTE* match = base + matchIndex; -#if LIZARD_NOCHAIN_MIN_OFFSET > 0 - if ((U32)(ip - match) >= LIZARD_NOCHAIN_MIN_OFFSET) -#endif - if (*(iLowLimit + longest) == *(match - LLdelta + longest)) { - if (MEM_read32(match) == MEM_read32(ip)) { - int mlt = MINMATCH + Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit); - int back = 0; - while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; - mlt -= back; - - if (mlt > longest) { - longest = (int)mlt; - *matchpos = match+back; - *startpos = ip+back; - } - } - } - } else { - const BYTE* match = dictBase + matchIndex; -#if LIZARD_NOCHAIN_MIN_OFFSET > 0 - if ((U32)(ip - (base + matchIndex)) >= LIZARD_NOCHAIN_MIN_OFFSET) -#endif - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(match) == MEM_read32(ip)) { - int back=0; - size_t mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; - while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; - mlt -= back; - if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } - } - } - } - - return longest; -} - - -FORCE_INLINE int Lizard_compress_noChain ( - Lizard_stream_t* const ctx, - const BYTE* ip, - const BYTE* const iend) -{ - const BYTE* anchor = ip; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); - - int ml, ml2, ml3, ml0; - const BYTE* ref = NULL; - const BYTE* start2 = NULL; - const BYTE* ref2 = NULL; - const BYTE* start3 = NULL; - const BYTE* ref3 = NULL; - const BYTE* start0; - const BYTE* ref0; - - /* init */ - ip++; - - /* Main Loop */ - while (ip < mflimit) { - ml = Lizard_InsertAndFindBestMatchNoChain (ctx, ip, matchlimit, (&ref)); - if (!ml) { ip++; continue; } - - /* saved, in case we would skip too much */ - start0 = ip; - ref0 = ref; - ml0 = ml; - -_Search2: - if (ip+ml < mflimit) - ml2 = Lizard_InsertAndGetWiderMatchNoChain(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2); - else ml2 = ml; - - if (ml2 == ml) { /* No better match */ - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; - continue; - } - - if (start0 < ip) { - if (start2 < ip + ml0) { /* empirical */ - ip = start0; - ref = ref0; - ml = ml0; - } - } - - /* Here, start0==ip */ - if ((start2 - ip) < 3) { /* First Match too small : removed */ - ml = ml2; - ip = start2; - ref =ref2; - goto _Search2; - } - -_Search3: - /* - * Currently we have : - * ml2 > ml1, and - * ip1+3 <= ip2 (usually < ip1+ml1) - */ - if ((start2 - ip) < OPTIMAL_ML) { - int correction; - int new_ml = ml; - if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; - if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; - correction = new_ml - (int)(start2 - ip); - if (correction > 0) { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } - /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ - - if (start2 + ml2 < mflimit) - ml3 = Lizard_InsertAndGetWiderMatchNoChain(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3); - else ml3 = ml2; - - if (ml3 == ml2) { /* No better match : 2 sequences to encode */ - /* ip & ref are known; Now for ml */ - if (start2 < ip+ml) ml = (int)(start2 - ip); - /* Now, encode 2 sequences */ - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; - ip = start2; - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml2, ref2)) return 0; - continue; - } - - if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */ - if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ - if (start2 < ip+ml) { - int correction = (int)(ip+ml - start2); - start2 += correction; - ref2 += correction; - ml2 -= correction; - if (ml2 < MINMATCH) { - start2 = start3; - ref2 = ref3; - ml2 = ml3; - } - } - - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; - ip = start3; - ref = ref3; - ml = ml3; - - start0 = start2; - ref0 = ref2; - ml0 = ml2; - goto _Search2; - } - - start2 = start3; - ref2 = ref3; - ml2 = ml3; - goto _Search3; - } - - /* - * OK, now we have 3 ascending matches; let's write at least the first one - * ip & ref are known; Now for ml - */ - if (start2 < ip+ml) { - if ((start2 - ip) < (int)ML_MASK_LZ4) { - int correction; - if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; - if (ip + ml > start2 + ml2 - MINMATCH) { - ml = (int)(start2 - ip) + ml2 - MINMATCH; - if (ml < MINMATCH) { // match2 doesn't fit, remove it - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; - ip = start3; - ref = ref3; - ml = ml3; - - start0 = start2; - ref0 = ref2; - ml0 = ml2; - goto _Search2; - } - } - correction = ml - (int)(start2 - ip); - if (correction > 0) { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } else { - ml = (int)(start2 - ip); - } - } - if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0; - - ip = start2; - ref = ref2; - ml = ml2; - - start2 = start3; - ref2 = ref3; - ml2 = ml3; - - goto _Search3; - } - - /* Encode Last Literals */ - ip = iend; - if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error; - - /* End */ - return 1; -_output_error: - return 0; -} diff --git a/contrib/lizard/lib/lizard_parser_optimal.h b/contrib/lizard/lib/lizard_parser_optimal.h deleted file mode 100644 index 8c9b83e9590..00000000000 --- a/contrib/lizard/lib/lizard_parser_optimal.h +++ /dev/null @@ -1,679 +0,0 @@ -#define LIZARD_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LIZARD_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) -#define LIZARD_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) - -#define LIZARD_OPTIMAL_MIN_OFFSET 8 -#define LIZARD_OPT_NUM (1<<12) -#define REPMINMATCH 1 - - -FORCE_INLINE size_t Lizard_get_price(Lizard_stream_t* const ctx, int rep, const BYTE *ip, const BYTE *off24pos, size_t litLength, U32 offset, size_t matchLength) -{ - if (ctx->params.decompressType == Lizard_coderwords_LZ4) - return Lizard_get_price_LZ4(ctx, ip, litLength, offset, matchLength); - - return Lizard_get_price_LIZv1(ctx, rep, ip, off24pos, litLength, offset, matchLength); -} - - - -typedef struct -{ - int off; - int len; - int back; -} Lizard_match_t; - -typedef struct -{ - int price; - int off; - int mlen; - int litlen; - int rep; - const BYTE* off24pos; -} Lizard_optimal_t; - - -/* Update chains up to ip (excluded) */ -FORCE_INLINE void Lizard_BinTree_Insert(Lizard_stream_t* ctx, const BYTE* ip) -{ -#if MINMATCH == 3 - U32* HashTable3 = ctx->hashTable3; - const BYTE* const base = ctx->base; - const U32 target = (U32)(ip - base); - U32 idx = ctx->nextToUpdate; - - while(idx < target) { - HashTable3[Lizard_hash3Ptr(base+idx, ctx->params.hashLog3)] = idx; - idx++; - } - - ctx->nextToUpdate = target; -#else - (void)ctx; (void)ip; -#endif -} - - - -FORCE_INLINE int Lizard_GetAllMatches ( - Lizard_stream_t* ctx, - const BYTE* const ip, - const BYTE* const iLowLimit, - const BYTE* const iHighLimit, - size_t best_mlen, - Lizard_match_t* matches) -{ - U32* const chainTable = ctx->chainTable; - U32* const HashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - const BYTE* const dictBase = ctx->dictBase; - const intptr_t dictLimit = ctx->dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1; - const intptr_t current = (ip - base); - const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance; - const U32 contentMask = (1 << ctx->params.contentLog) - 1; - const BYTE* match, *matchDict; - const size_t minMatchLongOff = ctx->params.minMatchLongOff; - intptr_t matchIndex; - int nbAttempts = ctx->params.searchNum; - // bool fullSearch = (ctx->params.fullSearch >= 2); - int mnum = 0; - U32* HashPos; - size_t mlt; - - if (ip + MINMATCH > iHighLimit) return 0; - - /* First Match */ - HashPos = &HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; - matchIndex = *HashPos; -#if MINMATCH == 3 - { - U32* const HashTable3 = ctx->hashTable3; - U32* HashPos3 = &HashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; - - if ((*HashPos3 < current) && (*HashPos3 >= lowLimit)) { - size_t offset = current - *HashPos3; - if (offset < LIZARD_MAX_8BIT_OFFSET) { - match = ip - offset; - if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) { - size_t mlt = Lizard_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; - - int back = 0; - while ((ip + back > iLowLimit) && (match + back > lowPrefixPtr) && (ip[back - 1] == match[back - 1])) back--; - mlt -= back; - - matches[mnum].off = (int)offset; - matches[mnum].len = (int)mlt; - matches[mnum].back = -back; - mnum++; - } - } - } - - *HashPos3 = current; - } -#endif - - chainTable[current & contentMask] = (U32)(current - matchIndex); - *HashPos = (U32)current; - ctx->nextToUpdate++; - - if (best_mlen < MINMATCH-1) best_mlen = MINMATCH-1; - - while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { - nbAttempts--; - match = base + matchIndex; - if ((U32)(ip - match) >= LIZARD_OPTIMAL_MIN_OFFSET) { - if (matchIndex >= dictLimit) { - if ((/*fullSearch ||*/ ip[best_mlen] == match[best_mlen]) && (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip))) { - int back = 0; - mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; - while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; - mlt -= back; - - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - if (mlt > best_mlen) { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - matches[mnum].back = -back; - mnum++; - - if (best_mlen > LIZARD_OPT_NUM) break; - } - } - } else { - matchDict = dictBase + matchIndex; - // fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match)); - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_readMINMATCH(matchDict) == MEM_readMINMATCH(ip)) { - int back=0; - mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH; - while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchDict[back-1])) back--; - mlt -= back; - - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - if (mlt > best_mlen) { - best_mlen = mlt; - matches[mnum].off = (int)(ip - match); - matches[mnum].len = (int)mlt; - matches[mnum].back = -back; - mnum++; - - if (best_mlen > LIZARD_OPT_NUM) break; - } - } - } - } - matchIndex -= chainTable[matchIndex & contentMask]; - } - - return mnum; -} - - - - -FORCE_INLINE int Lizard_BinTree_GetAllMatches ( - Lizard_stream_t* ctx, - const BYTE* const ip, - const BYTE* const iHighLimit, - size_t best_mlen, - Lizard_match_t* matches) -{ - U32* const chainTable = ctx->chainTable; - U32* const HashTable = ctx->hashTable; - const BYTE* const base = ctx->base; - const intptr_t dictLimit = ctx->dictLimit; - const BYTE* const dictBase = ctx->dictBase; - const BYTE* const dictEnd = dictBase + dictLimit; - const U32 contentMask = (1 << ctx->params.contentLog) - 1; - const BYTE* const lowPrefixPtr = base + dictLimit; - const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1; - const intptr_t current = (ip - base); - const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance; - const BYTE* match; - const size_t minMatchLongOff = ctx->params.minMatchLongOff; - int nbAttempts = ctx->params.searchNum; - int mnum = 0; - U32 *ptr0, *ptr1, delta0, delta1; - intptr_t matchIndex; - size_t mlt = 0; - U32* HashPos; - - if (ip + MINMATCH > iHighLimit) return 0; - - /* First Match */ - HashPos = &HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; - matchIndex = *HashPos; - - -#if MINMATCH == 3 - { - U32* HashPos3 = &ctx->hashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; - - if ((*HashPos3 < current) && (*HashPos3 >= lowLimit)) { - size_t offset = current - *HashPos3; - if (offset < LIZARD_MAX_8BIT_OFFSET) { - match = ip - offset; - if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) - { - mlt = Lizard_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; - - matches[mnum].off = (int)offset; - matches[mnum].len = (int)mlt; - matches[mnum].back = 0; - mnum++; - } - } - *HashPos3 = current; - } - } -#endif - - *HashPos = (U32)current; - ctx->nextToUpdate++; - - // check rest of matches - ptr0 = &chainTable[(current*2+1) & contentMask]; - ptr1 = &chainTable[(current*2) & contentMask]; - delta0 = delta1 = (U32)(current - matchIndex); - - if (best_mlen < MINMATCH-1) best_mlen = MINMATCH-1; - - while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) { - nbAttempts--; - if (matchIndex >= dictLimit) { - match = base + matchIndex; - // if (ip[mlt] == match[mlt]) - mlt = Lizard_count(ip, match, iHighLimit); - } else { - match = dictBase + matchIndex; - mlt = Lizard_count_2segments(ip, match, iHighLimit, dictEnd, lowPrefixPtr); - if (matchIndex + (int)mlt >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[mlt] */ - } - - if ((U32)(current - matchIndex) >= LIZARD_OPTIMAL_MIN_OFFSET) { - if ((mlt >= minMatchLongOff) || ((U32)(current - matchIndex) < LIZARD_MAX_16BIT_OFFSET)) - if (mlt > best_mlen) { - best_mlen = mlt; - matches[mnum].off = (int)(current - matchIndex); - matches[mnum].len = (int)mlt; - matches[mnum].back = 0; - mnum++; - - if (mlt > LIZARD_OPT_NUM) break; - if (ip + mlt >= iHighLimit) break; - } - } else { -#if 1 - intptr_t newMatchIndex; - size_t newml = 0, newoff = 0; - do { - newoff += (int)(current - matchIndex); - } while (newoff < LIZARD_OPTIMAL_MIN_OFFSET); - newMatchIndex = current - newoff; - if (newMatchIndex >= dictLimit) newml = Lizard_count(ip, base + newMatchIndex, iHighLimit); - - // printf("%d: off=%d mlt=%d\n", (U32)current, (U32)(current - matchIndex), (int)mlt); - // printf("%d: newoff=%d newml=%d\n", (U32)current, (int)newoff, (int)newml); - - if ((newml >= minMatchLongOff) && (newml > best_mlen)) { - best_mlen = newml; - matches[mnum].off = (int)newoff; - matches[mnum].len = (int)newml; - matches[mnum].back = 0; - mnum++; - - if (newml > LIZARD_OPT_NUM) break; - if (ip + newml >= iHighLimit) break; - } -#endif - } - - if (ip[mlt] < match[mlt]) { - *ptr0 = delta0; - ptr0 = &chainTable[(matchIndex*2) & contentMask]; - if (*ptr0 == (U32)-1) break; - delta0 = *ptr0; - delta1 += delta0; - matchIndex -= delta0; - } else { - *ptr1 = delta1; - ptr1 = &chainTable[(matchIndex*2+1) & contentMask]; - if (*ptr1 == (U32)-1) break; - delta1 = *ptr1; - delta0 += delta1; - matchIndex -= delta1; - } - } - - *ptr0 = (U32)-1; - *ptr1 = (U32)-1; - - return mnum; -} - - -#define SET_PRICE(pos, mlen, offset, litlen, price) \ - { \ - while (last_pos < pos) { opt[last_pos+1].price = LIZARD_MAX_PRICE; last_pos++; } \ - opt[pos].mlen = (int)mlen; \ - opt[pos].off = (int)offset; \ - opt[pos].litlen = (int)litlen; \ - opt[pos].price = (int)price; \ - LIZARD_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ - } - - -FORCE_INLINE int Lizard_compress_optimalPrice( - Lizard_stream_t* const ctx, - const BYTE* ip, - const BYTE* const iend) -{ - Lizard_optimal_t opt[LIZARD_OPT_NUM + 4]; - Lizard_match_t matches[LIZARD_OPT_NUM + 1]; - const BYTE *inr; - size_t res, cur, cur2, skip_num = 0; - size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; - - const BYTE* anchor = ip; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); - const BYTE* const base = ctx->base; - const BYTE* const dictBase = ctx->dictBase; - const intptr_t dictLimit = ctx->dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const lowPrefixPtr = base + dictLimit; - const intptr_t lowLimit = ctx->lowLimit; - const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1; - - const size_t sufficient_len = ctx->params.sufficientLength; - const int faster_get_matches = (ctx->params.fullSearch == 0); - const size_t minMatchLongOff = ctx->params.minMatchLongOff; - const int lizardOptimalMinOffset = (ctx->params.decompressType == Lizard_coderwords_LZ4) ? (1<<30) : LIZARD_OPTIMAL_MIN_OFFSET; - const size_t repMinMatch = (ctx->params.decompressType == Lizard_coderwords_LZ4) ? MINMATCH : REPMINMATCH; - - /* Main Loop */ - while (ip < mflimit) { - memset(opt, 0, sizeof(Lizard_optimal_t)); - last_pos = 0; - llen = ip - anchor; - - /* check rep code */ - - if (ctx->last_off >= lizardOptimalMinOffset) { - intptr_t matchIndexLO = (ip - ctx->last_off) - base; - mlen = 0; - if ((matchIndexLO >= lowLimit) && (base + matchIndexLO + maxDistance >= ip)) { - if (matchIndexLO >= dictLimit) { - mlen = Lizard_count(ip, base + matchIndexLO, matchlimit); - } else { - mlen = Lizard_count_2segments(ip, dictBase + matchIndexLO, matchlimit, dictEnd, lowPrefixPtr); - } - } - if (mlen >= REPMINMATCH) { - if (mlen > sufficient_len || mlen >= LIZARD_OPT_NUM) { - best_mlen = mlen; best_off = 0; cur = 0; last_pos = 1; - goto encode; - } - - do - { - litlen = 0; - price = Lizard_get_price(ctx, ctx->last_off, ip, ctx->off24pos, llen, 0, mlen); - if (mlen > last_pos || price < (size_t)opt[mlen].price) - SET_PRICE(mlen, mlen, 0, litlen, price); - mlen--; - } - while (mlen >= REPMINMATCH); - } - } - - if (faster_get_matches && last_pos) - match_num = 0; - else - { - if (ctx->params.parserType == Lizard_parser_optimalPrice) { - Lizard_Insert(ctx, ip); - match_num = Lizard_GetAllMatches(ctx, ip, ip, matchlimit, last_pos, matches); - } else { - Lizard_BinTree_Insert(ctx, ip); - match_num = Lizard_BinTree_GetAllMatches(ctx, ip, matchlimit, last_pos, matches); - } - } - - LIZARD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); - if (!last_pos && !match_num) { ip++; continue; } - - if (match_num && (size_t)matches[match_num-1].len > sufficient_len) { - best_mlen = matches[match_num-1].len; - best_off = matches[match_num-1].off; - cur = 0; - last_pos = 1; - goto encode; - } - - // set prices using matches at position = 0 - best_mlen = (last_pos > MINMATCH) ? last_pos : MINMATCH; - - for (i = 0; i < match_num; i++) { - mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; - best_mlen = (matches[i].len < LIZARD_OPT_NUM) ? matches[i].len : LIZARD_OPT_NUM; - LIZARD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); - while (mlen <= best_mlen){ - litlen = 0; - price = Lizard_get_price(ctx, ctx->last_off, ip, ctx->off24pos, llen + litlen, matches[i].off, mlen); - - if ((mlen >= minMatchLongOff) || (matches[i].off < LIZARD_MAX_16BIT_OFFSET)) - if (mlen > last_pos || price < (size_t)opt[mlen].price) - SET_PRICE(mlen, mlen, matches[i].off, litlen, price); - mlen++; - } - } - - if (last_pos < repMinMatch) { ip++; continue; } - - opt[0].off24pos = ctx->off24pos; - opt[0].rep = ctx->last_off; - opt[0].mlen = 1; - opt[0].off = -1; - - // check further positions - for (skip_num = 0, cur = 1; cur <= last_pos; cur++) { - int rep; - inr = ip + cur; - - if (opt[cur-1].off == -1) { // -1 = literals, 0 = rep - litlen = opt[cur-1].litlen + 1; - - if (cur != litlen) { - price = opt[cur - litlen].price + Lizard_get_price(ctx, opt[cur-litlen].rep, inr, ctx->off24pos, litlen, 0, 0); - LIZARD_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); - } else { - price = Lizard_get_price(ctx, ctx->last_off, inr, ctx->off24pos, llen + litlen, 0, 0); - LIZARD_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); - } - } else { - litlen = 1; - price = opt[cur - 1].price + Lizard_get_price(ctx, opt[cur-1].rep, inr, ctx->off24pos, litlen, 0, 0); - LIZARD_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, Lizard_get_price(ctx, rep, inr, ctx->off24pos, litlen, 0, 0)); - } - - mlen = 1; - best_mlen = 0; - LIZARD_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); - - if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) - SET_PRICE(cur, mlen, -1, litlen, price); - - if (cur == last_pos) break; - - - - /* set rep code */ - if (opt[cur].off != -1) { - mlen = opt[cur].mlen; - offset = opt[cur].off; - if (offset < 1) { - opt[cur].rep = opt[cur-mlen].rep; - opt[cur].off24pos = opt[cur-mlen].off24pos; - LIZARD_LOG_PARSER("%d: COPYREP1 cur=%d mlen=%d rep=%d\n", (int)(inr-source), cur, mlen, opt[cur-mlen].rep); - } else { - opt[cur].rep = (int)offset; - opt[cur].off24pos = (offset >= LIZARD_MAX_16BIT_OFFSET) ? inr : opt[cur-mlen].off24pos; - LIZARD_LOG_PARSER("%d: COPYREP2 cur=%d offset=%d rep=%d\n", (int)(inr-source), cur, offset, opt[cur].rep); - } - } else { - opt[cur].rep = opt[cur-1].rep; // copy rep - opt[cur].off24pos = opt[cur-1].off24pos; - } - - rep = opt[cur].rep; - LIZARD_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep); - - - /* check rep code */ - if (opt[cur].rep >= lizardOptimalMinOffset) { - intptr_t matchIndexLO = (inr - opt[cur].rep) - base; - mlen = 0; - if ((matchIndexLO >= lowLimit) && (base + matchIndexLO + maxDistance >= inr)) { - if (matchIndexLO >= dictLimit) { - mlen = Lizard_count(inr, base + matchIndexLO, matchlimit); - } else { - mlen = Lizard_count_2segments(inr, dictBase + matchIndexLO, matchlimit, dictEnd, lowPrefixPtr); - } - } - if (mlen >= REPMINMATCH/* && mlen > best_mlen*/) { - LIZARD_LOG_PARSER("%d: try REP rep=%d mlen=%d\n", (int)(inr-source), opt[cur].rep, mlen); - LIZARD_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-source), mlen, 0, opt[cur].rep, cur, opt[cur].off); - - if (mlen > sufficient_len || cur + mlen >= LIZARD_OPT_NUM) { - best_mlen = mlen; - best_off = 0; - LIZARD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-source), sufficient_len, best_mlen, best_off, last_pos); - last_pos = cur + 1; - goto encode; - } - - best_mlen = mlen; - if (faster_get_matches) - skip_num = best_mlen; - - do - { - //if (opt[cur].mlen == 1) - if (opt[cur].off == -1) { - litlen = opt[cur].litlen; - - if (cur != litlen) { - price = opt[cur - litlen].price + Lizard_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen); - LIZARD_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); - } else { - price = Lizard_get_price(ctx, rep, inr, ctx->off24pos, llen + litlen, 0, mlen); - LIZARD_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); - } - } else { - litlen = 0; - price = opt[cur].price + Lizard_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen); - LIZARD_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d getprice=%d\n", (int)(inr-source), price, cur, litlen, Lizard_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen - MINMATCH)); - } - - LIZARD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, 0, price, litlen, cur - litlen, opt[cur - litlen].price); - - if (cur + mlen > last_pos || price <= (size_t)opt[cur + mlen].price) // || ((price == opt[cur + mlen].price) && (opt[cur].mlen == 1) && (cur != litlen))) // at equal price prefer REP instead of MATCH - SET_PRICE(cur + mlen, mlen, 0, litlen, price); - mlen--; - } - while (mlen >= REPMINMATCH); - } - } - - if (faster_get_matches && skip_num > 0) { - skip_num--; - continue; - } - - if (ctx->params.parserType == Lizard_parser_optimalPrice) { - Lizard_Insert(ctx, inr); - match_num = Lizard_GetAllMatches(ctx, inr, ip, matchlimit, best_mlen, matches); - LIZARD_LOG_PARSER("%d: Lizard_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); - } else { - Lizard_BinTree_Insert(ctx, inr); - match_num = Lizard_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); - LIZARD_LOG_PARSER("%d: Lizard_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); - } - - - if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { - cur -= matches[match_num-1].back; - best_mlen = matches[match_num-1].len; - best_off = matches[match_num-1].off; - last_pos = cur + 1; - goto encode; - } - - // set prices using matches at position = cur - best_mlen = (best_mlen > MINMATCH) ? best_mlen : MINMATCH; - - for (i = 0; i < match_num; i++) { - mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; - cur2 = cur - matches[i].back; - best_mlen = (cur2 + matches[i].len < LIZARD_OPT_NUM) ? (size_t)matches[i].len : LIZARD_OPT_NUM - cur2; - LIZARD_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); - - if (mlen < (size_t)matches[i].back + 1) - mlen = matches[i].back + 1; - - while (mlen <= best_mlen) { - // if (opt[cur2].mlen == 1) - if (opt[cur2].off == -1) - { - litlen = opt[cur2].litlen; - - if (cur2 != litlen) - price = opt[cur2 - litlen].price + Lizard_get_price(ctx, rep, inr, opt[cur2].off24pos, litlen, matches[i].off, mlen); - else - price = Lizard_get_price(ctx, rep, inr, ctx->off24pos, llen + litlen, matches[i].off, mlen); - } else { - litlen = 0; - price = opt[cur2].price + Lizard_get_price(ctx, rep, inr, opt[cur2].off24pos, litlen, matches[i].off, mlen); - } - - LIZARD_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); - // if (cur2 + mlen > last_pos || ((matches[i].off != opt[cur2 + mlen].off) && (price < opt[cur2 + mlen].price))) - - if ((mlen >= minMatchLongOff) || (matches[i].off < LIZARD_MAX_16BIT_OFFSET)) - if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) - { - SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); - } - - mlen++; - } - } - } // for (skip_num = 0, cur = 1; cur <= last_pos; cur++) - - - best_mlen = opt[last_pos].mlen; - best_off = opt[last_pos].off; - cur = last_pos - best_mlen; - - encode: // cur, last_pos, best_mlen, best_off have to be set - for (i = 1; i <= last_pos; i++) { - LIZARD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep); - } - - LIZARD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off, opt[cur].rep); - - opt[0].mlen = 1; - - while (1) { - mlen = opt[cur].mlen; - offset = opt[cur].off; - opt[cur].mlen = (int)best_mlen; - opt[cur].off = (int)best_off; - best_mlen = mlen; - best_off = offset; - if (mlen > cur) break; - cur -= mlen; - } - - for (i = 0; i <= last_pos;) { - LIZARD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep); - i += opt[i].mlen; - } - - cur = 0; - - while (cur < last_pos) { - LIZARD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep); - mlen = opt[cur].mlen; - // if (mlen == 1) { ip++; cur++; continue; } - if (opt[cur].off == -1) { ip++; cur++; continue; } - offset = opt[cur].off; - cur += mlen; - - LIZARD_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-source), (int)(ip-anchor), (int)(offset), mlen); - res = Lizard_encodeSequence(ctx, &ip, &anchor, mlen, ip - offset); - if (res) return 0; - - LIZARD_LOG_PARSER("%d: offset=%d rep=%d\n", (int)(ip-source), offset, ctx->last_off); - } - } - - /* Encode Last Literals */ - ip = iend; - if (Lizard_encodeLastLiterals(ctx, &ip, &anchor)) goto _output_error; - - /* End */ - return 1; -_output_error: - return 0; -} - diff --git a/contrib/lizard/lib/lizard_parser_pricefast.h b/contrib/lizard/lib/lizard_parser_pricefast.h deleted file mode 100644 index 9818e2c4212..00000000000 --- a/contrib/lizard/lib/lizard_parser_pricefast.h +++ /dev/null @@ -1,250 +0,0 @@ -#define LIZARD_PRICEFAST_MIN_OFFSET 8 - -FORCE_INLINE int Lizard_FindMatchFast(Lizard_stream_t* ctx, intptr_t matchIndex, intptr_t matchIndex3, /* Index table will be updated */ - const BYTE* ip, const BYTE* const iLimit, - const BYTE** matchpos) -{ - const BYTE* const base = ctx->base; - const BYTE* const dictBase = ctx->dictBase; - const intptr_t dictLimit = ctx->dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1; - const intptr_t current = (U32)(ip - base); - const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance; - const BYTE* const lowPrefixPtr = base + dictLimit; - const size_t minMatchLongOff = ctx->params.minMatchLongOff; - const BYTE* match, *matchDict; - size_t ml=0, mlt; - - if (ctx->last_off >= LIZARD_PRICEFAST_MIN_OFFSET) { - intptr_t matchIndexLO = (ip - ctx->last_off) - base; - if (matchIndexLO >= lowLimit) { - if (matchIndexLO >= dictLimit) { - match = base + matchIndexLO; - if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) { - mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) - { - *matchpos = match; - return (int)mlt; - } - } - } else { - match = dictBase + matchIndexLO; - if ((U32)((dictLimit-1) - matchIndexLO) >= 3) /* intentional overflow */ - if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) { - mlt = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; - // if ((mlt >= minMatchLongOff) || (ctx->last_off < LIZARD_MAX_16BIT_OFFSET)) - { - *matchpos = base + matchIndexLO; /* virtual matchpos */ - return (int)mlt; - } - } - } - } - } - - -#if MINMATCH == 3 - if (matchIndex3 < current && matchIndex3 >= lowLimit) { - intptr_t offset = current - matchIndex3; - if (offset < LIZARD_MAX_8BIT_OFFSET) { - match = ip - offset; - if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) { - ml = 3;//Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - *matchpos = match; - } - } - } -#else - (void)matchIndex3; -#endif - - if ((matchIndex < current) && (matchIndex >= lowLimit)) { - match = base + matchIndex; - if ((U32)(ip - match) >= LIZARD_PRICEFAST_MIN_OFFSET) { - if (matchIndex >= dictLimit) { - if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) { - mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - if (!ml || (mlt > ml)) // && Lizard_better_price((ip - *matchpos), ml, (ip - match), mlt, ctx->last_off))) - { ml = mlt; *matchpos = match; } - } - } else { - matchDict = dictBase + matchIndex; - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(matchDict) == MEM_read32(ip)) { - mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - if (!ml || (mlt > ml)) // && Lizard_better_price((ip - *matchpos), ml, (U32)(ip - match), mlt, ctx->last_off))) - { ml = mlt; *matchpos = match; } /* virtual matchpos */ - } - } - } - } - - return (int)ml; -} - - -FORCE_INLINE int Lizard_FindMatchFaster (Lizard_stream_t* ctx, U32 matchIndex, /* Index table will be updated */ - const BYTE* ip, const BYTE* const iLimit, - const BYTE** matchpos) -{ - const BYTE* const base = ctx->base; - const BYTE* const dictBase = ctx->dictBase; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const U32 maxDistance = (1 << ctx->params.windowLog) - 1; - const U32 current = (U32)(ip - base); - const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance; - const BYTE* const lowPrefixPtr = base + dictLimit; - const size_t minMatchLongOff = ctx->params.minMatchLongOff; - const BYTE* match, *matchDict; - size_t ml=0, mlt; - - if (matchIndex < current && matchIndex >= lowLimit) { - match = base + matchIndex; - if ((U32)(ip - match) >= LIZARD_PRICEFAST_MIN_OFFSET) { - if (matchIndex >= dictLimit) { - if (MEM_read32(match) == MEM_read32(ip)) { - mlt = Lizard_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - { ml = mlt; *matchpos = match; } - } - } else { - matchDict = dictBase + matchIndex; - if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */ - if (MEM_read32(matchDict) == MEM_read32(ip)) { - mlt = Lizard_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH; - if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET)) - { ml = mlt; *matchpos = match; } /* virtual matchpos */ - } - } - } - } - - return (int)ml; -} - - - -FORCE_INLINE int Lizard_compress_priceFast( - Lizard_stream_t* const ctx, - const BYTE* ip, - const BYTE* const iend) -{ - const BYTE* anchor = ip; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); - - size_t ml, ml2=0; - const BYTE* ref=NULL; - const BYTE* start2=NULL; - const BYTE* ref2=NULL; - const BYTE* lowPrefixPtr = ctx->base + ctx->dictLimit; - U32* HashTable = ctx->hashTable; -#if MINMATCH == 3 - U32* HashTable3 = ctx->hashTable3; -#endif - const BYTE* const base = ctx->base; - const size_t minMatchLongOff = ctx->params.minMatchLongOff; - U32* HashPos; - - /* init */ - ip++; - - /* Main Loop */ - while (ip < mflimit) - { - HashPos = &HashTable[Lizard_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; -#if MINMATCH == 3 - { - U32* HashPos3 = &HashTable3[Lizard_hash3Ptr(ip, ctx->params.hashLog3)]; - ml = Lizard_FindMatchFast (ctx, *HashPos, *HashPos3, ip, matchlimit, (&ref)); - *HashPos3 = (U32)(ip - base); - } -#else - ml = Lizard_FindMatchFast (ctx, *HashPos, 0, ip, matchlimit, (&ref)); -#endif - if ((*HashPos >= (U32)(ip - base)) || ((U32)(ip - base) >= *HashPos + LIZARD_PRICEFAST_MIN_OFFSET)) - *HashPos = (U32)(ip - base); - - if (!ml) { ip++; continue; } - if ((int)(ip - ref) == ctx->last_off) { ml2=0; ref=ip; goto _Encode; } - - { - int back = 0; - while ((ip+back>anchor) && (ref+back > lowPrefixPtr) && (ip[back-1] == ref[back-1])) back--; - ml -= back; - ip += back; - ref += back; - } - -_Search: - if (ip+ml >= mflimit) goto _Encode; - - start2 = ip + ml - 2; - HashPos = &HashTable[Lizard_hashPtr(start2, ctx->params.hashLog, ctx->params.searchLength)]; - ml2 = Lizard_FindMatchFaster(ctx, *HashPos, start2, matchlimit, (&ref2)); - if ((*HashPos >= (U32)(start2 - base)) || ((U32)(start2 - base) >= *HashPos + LIZARD_PRICEFAST_MIN_OFFSET)) - *HashPos = (U32)(start2 - base); - - if (!ml2) goto _Encode; - - { - int back = 0; - while ((start2+back>ip) && (ref2+back > lowPrefixPtr) && (start2[back-1] == ref2[back-1])) back--; - ml2 -= back; - start2 += back; - ref2 += back; - } - - if (ml2 <= ml) { ml2 = 0; goto _Encode; } - - if (start2 <= ip) - { - - ip = start2; ref = ref2; ml = ml2; - ml2 = 0; - goto _Encode; - } - - if (start2 - ip < 3) - { - ip = start2; ref = ref2; ml = ml2; - ml2 = 0; - goto _Search; - } - - if (start2 < ip + ml) - { - size_t correction = ml - (int)(start2 - ip); - start2 += correction; - ref2 += correction; - ml2 -= correction; - if (ml2 < 3) { ml2 = 0; } - if ((ml2 < minMatchLongOff) && ((U32)(start2 - ref2) >= LIZARD_MAX_16BIT_OFFSET)) { ml2 = 0; } - } - -_Encode: - if (Lizard_encodeSequence_LIZv1(ctx, &ip, &anchor, ml, ref)) goto _output_error; - - if (ml2) - { - ip = start2; ref = ref2; ml = ml2; - ml2 = 0; - goto _Search; - } - } - - /* Encode Last Literals */ - ip = iend; - if (Lizard_encodeLastLiterals_LIZv1(ctx, &ip, &anchor)) goto _output_error; - - /* End */ - return 1; -_output_error: - return 0; -} - diff --git a/contrib/lizard/lib/xxhash/xxhash.c b/contrib/lizard/lib/xxhash/xxhash.c deleted file mode 100644 index 833b99f3b21..00000000000 --- a/contrib/lizard/lib/xxhash/xxhash.c +++ /dev/null @@ -1,888 +0,0 @@ -/* -* xxHash - Fast Hash algorithm -* Copyright (C) 2012-2016, Yann Collet -* -* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions are -* met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following disclaimer -* in the documentation and/or other materials provided with the -* distribution. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -* -* You can contact the author at : -* - xxHash homepage: http://www.xxhash.com -* - xxHash source repository : https://github.com/Cyan4973/xxHash -*/ - - -/* ************************************* -* Tuning parameters -***************************************/ -/*!XXH_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. - * It can generate buggy code on targets which do not support unaligned memory accesses. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://stackoverflow.com/a/32095106/646947 for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define XXH_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) -# define XXH_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -/*!XXH_ACCEPT_NULL_INPUT_POINTER : - * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. - * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. - * By default, this option is disabled. To enable it, uncomment below define : - */ -/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ - -/*!XXH_FORCE_NATIVE_FORMAT : - * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. - * Results are therefore identical for little-endian and big-endian CPU. - * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. - * Should endian-independence be of no importance for your application, you may set the #define below to 1, - * to improve speed for Big-endian CPU. - * This option has no impact on Little_Endian CPU. - */ -#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ -# define XXH_FORCE_NATIVE_FORMAT 0 -#endif - -/*!XXH_FORCE_ALIGN_CHECK : - * This is a minor performance trick, only useful with lots of very small keys. - * It means : check for aligned/unaligned input. - * The check costs one initial branch per hash; - * set it to 0 when the input is guaranteed to be aligned, - * or when alignment doesn't matter for performance. - */ -#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ -# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_FORCE_ALIGN_CHECK 0 -# else -# define XXH_FORCE_ALIGN_CHECK 1 -# endif -#endif - - -/* ************************************* -* Includes & Memory related functions -***************************************/ -/*! Modify the local functions below should you wish to use some other memory routines -* for malloc(), free() */ -#include -static void* XXH_malloc(size_t s) { return malloc(s); } -static void XXH_free (void* p) { free(p); } -/*! and for memcpy() */ -#include -static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } - -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" - - -/* ************************************* -* Compiler Specific Options -***************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# define FORCE_INLINE static __forceinline -#else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -# else -# define FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -#endif - - -/* ************************************* -* Basic Types -***************************************/ -#ifndef MEM_MODULE -# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; -# else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; -# endif -#endif - -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) - -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } - -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U32 u32; } __attribute__((packed)) unalign; -static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } - -#else - -/* portable and safe solution. Generally efficient. - * see : http://stackoverflow.com/a/32095106/646947 - */ -static U32 XXH_read32(const void* memPtr) -{ - U32 val; - memcpy(&val, memPtr, sizeof(val)); - return val; -} - -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - - -/* **************************************** -* Compiler-specific Functions and Macros -******************************************/ -#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) -#endif - -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap32 _byteswap_ulong -#elif XXH_GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -#else -static U32 XXH_swap32 (U32 x) -{ - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); -} -#endif - - -/* ************************************* -* Architecture Macros -***************************************/ -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; - -/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ -#ifndef XXH_CPU_LITTLE_ENDIAN - static const int g_one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) -#endif - - -/* *************************** -* Memory reads -*****************************/ -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); - else - return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); -} - -FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE32_align(ptr, endian, XXH_unaligned); -} - -static U32 XXH_readBE32(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); -} - - -/* ************************************* -* Macros -***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ -XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } - - -/* ******************************************************************* -* 32-bits hash functions -*********************************************************************/ -static const U32 PRIME32_1 = 2654435761U; -static const U32 PRIME32_2 = 2246822519U; -static const U32 PRIME32_3 = 3266489917U; -static const U32 PRIME32_4 = 668265263U; -static const U32 PRIME32_5 = 374761393U; - -static U32 XXH32_round(U32 seed, U32 input) -{ - seed += input * PRIME32_2; - seed = XXH_rotl32(seed, 13); - seed *= PRIME32_1; - return seed; -} - -FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)16; - } -#endif - - if (len>=16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do { - v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; - v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; - v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; - v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; - } while (p<=limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } else { - h32 = seed + PRIME32_5; - } - - h32 += (U32) len; - - while (p+4<=bEnd) { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_state_t state; - XXH32_reset(&state, seed); - XXH32_update(&state, input, len); - return XXH32_digest(&state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - - -/*====== Hash streaming ======*/ - -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) -{ - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) -{ - memcpy(dstState, srcState, sizeof(*dstState)); -} - -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) -{ - XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME32_1 + PRIME32_2; - state.v2 = seed + PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME32_1; - memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} - - -FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len_32 += (unsigned)len; - state->large_len |= (len>=16) | (state->total_len_32>=16); - - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (unsigned)len; - return XXH_OK; - } - - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const U32* p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do { - v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; - v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; - v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; - v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem32; - const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; - U32 h32; - - if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } else { - h32 = state->v3 /* == seed */ + PRIME32_5; - } - - h32 += state->total_len_32; - - while (p+4<=bEnd) { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_digest_endian(state_in, XXH_littleEndian); - else - return XXH32_digest_endian(state_in, XXH_bigEndian); -} - - -/*====== Canonical representation ======*/ - -/*! Default XXH result types are basic unsigned 32 and 64 bits. -* The canonical representation follows human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. -*/ - -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) -{ - return XXH_readBE32(src); -} - - -#ifndef XXH_NO_LONG_LONG - -/* ******************************************************************* -* 64-bits hash functions -*********************************************************************/ - -/*====== Memory access ======*/ - -#ifndef MEM_MODULE -# define MEM_MODULE -# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint64_t U64; -# else - typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */ -# endif -#endif - - -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) - -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } - -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64; -static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; } - -#else - -/* portable and safe solution. Generally efficient. - * see : http://stackoverflow.com/a/32095106/646947 - */ - -static U64 XXH_read64(const void* memPtr) -{ - U64 val; - memcpy(&val, memPtr, sizeof(val)); - return val; -} - -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap64 _byteswap_uint64 -#elif XXH_GCC_VERSION >= 403 -# define XXH_swap64 __builtin_bswap64 -#else -static U64 XXH_swap64 (U64 x) -{ - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); -} -#endif - -FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); - else - return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); -} - -FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE64_align(ptr, endian, XXH_unaligned); -} - -static U64 XXH_readBE64(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); -} - - -/*====== xxh64 ======*/ - -static const U64 PRIME64_1 = 11400714785074694791ULL; -static const U64 PRIME64_2 = 14029467366897019727ULL; -static const U64 PRIME64_3 = 1609587929392839161ULL; -static const U64 PRIME64_4 = 9650029242287828579ULL; -static const U64 PRIME64_5 = 2870177450012600261ULL; - -static U64 XXH64_round(U64 acc, U64 input) -{ - acc += input * PRIME64_2; - acc = XXH_rotl64(acc, 31); - acc *= PRIME64_1; - return acc; -} - -static U64 XXH64_mergeRound(U64 acc, U64 val) -{ - val = XXH64_round(0, val); - acc ^= val; - acc = acc * PRIME64_1 + PRIME64_4; - return acc; -} - -FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)32; - } -#endif - - if (len>=32) { - const BYTE* const limit = bEnd - 32; - U64 v1 = seed + PRIME64_1 + PRIME64_2; - U64 v2 = seed + PRIME64_2; - U64 v3 = seed + 0; - U64 v4 = seed - PRIME64_1; - - do { - v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; - v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; - v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; - v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; - } while (p<=limit); - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - - } else { - h64 = seed + PRIME64_5; - } - - h64 += (U64) len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_get64bits(p)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_state_t state; - XXH64_reset(&state, seed); - XXH64_update(&state, input, len); - return XXH64_digest(&state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - -/*====== Hash Streaming ======*/ - -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) -{ - return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) -{ - memcpy(dstState, srcState, sizeof(*dstState)); -} - -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) -{ - XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ - state.v1 = seed + PRIME64_1 + PRIME64_2; - state.v2 = seed + PRIME64_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME64_1; - memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} - -FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 32) { /* fill in tmp buffer */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); - p += 32-state->memsize; - state->memsize = 0; - } - - if (p+32 <= bEnd) { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - do { - v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; - v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; - v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; - v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); - state->memsize = (unsigned)(bEnd-p); - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH64_update_endian(state_in, input, len, XXH_bigEndian); -} - -FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem64; - const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; - U64 h64; - - if (state->total_len >= 32) { - U64 const v1 = state->v1; - U64 const v2 = state->v2; - U64 const v3 = state->v3; - U64 const v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - } else { - h64 = state->v3 + PRIME64_5; - } - - h64 += (U64) state->total_len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - -XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_digest_endian(state_in, XXH_littleEndian); - else - return XXH64_digest_endian(state_in, XXH_bigEndian); -} - - -/*====== Canonical representation ======*/ - -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) -{ - return XXH_readBE64(src); -} - -#endif /* XXH_NO_LONG_LONG */ diff --git a/contrib/lizard/lib/xxhash/xxhash.h b/contrib/lizard/lib/xxhash/xxhash.h deleted file mode 100644 index 9d831e03b35..00000000000 --- a/contrib/lizard/lib/xxhash/xxhash.h +++ /dev/null @@ -1,293 +0,0 @@ -/* - xxHash - Extremely Fast Hash algorithm - Header File - Copyright (C) 2012-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - xxHash source repository : https://github.com/Cyan4973/xxHash -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. - -A 64-bits version, named XXH64, is available since r35. -It offers much better speed, but for 64-bits applications only. -Name Speed on 64 bits Speed on 32 bits -XXH64 13.8 GB/s 1.9 GB/s -XXH32 6.8 GB/s 6.0 GB/s -*/ - -#ifndef XXHASH_H_5627135585666179 -#define XXHASH_H_5627135585666179 1 - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************** -* Definitions -******************************/ -#include /* size_t */ -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - -/* **************************** -* API modifier -******************************/ -/** XXH_PRIVATE_API -* This is useful to include xxhash functions in `static` mode -* in order to inline them, and remove their symbol from the public list. -* Methodology : -* #define XXH_PRIVATE_API -* #include "xxhash.h" -* `xxhash.c` is automatically included. -* It's not useful to compile and link it as a separate module. -*/ -#ifdef XXH_PRIVATE_API -# ifndef XXH_STATIC_LINKING_ONLY -# define XXH_STATIC_LINKING_ONLY -# endif -# if defined(__GNUC__) -# define XXH_PUBLIC_API static __inline __attribute__((unused)) -# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define XXH_PUBLIC_API static inline -# elif defined(_MSC_VER) -# define XXH_PUBLIC_API static __inline -# else -# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ -# endif -#else -# define XXH_PUBLIC_API /* do nothing */ -#endif /* XXH_PRIVATE_API */ - -/*!XXH_NAMESPACE, aka Namespace Emulation : - -If you want to include _and expose_ xxHash functions from within your own library, -but also want to avoid symbol collisions with other libraries which may also include xxHash, - -you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library -with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). - -Note that no change is required within the calling program as long as it includes `xxhash.h` : -regular symbol name will be automatically translated by this header. -*/ -#ifdef XXH_NAMESPACE -# define XXH_CAT(A,B) A##B -# define XXH_NAME2(A,B) XXH_CAT(A,B) -# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) -# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) -# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) -# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) -# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) -# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) -# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) -# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) -# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) -# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) -# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) -# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) -# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) -# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) -# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) -# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) -# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) -# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) -# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) -#endif - - -/* ************************************* -* Version -***************************************/ -#define XXH_VERSION_MAJOR 0 -#define XXH_VERSION_MINOR 6 -#define XXH_VERSION_RELEASE 2 -#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) -XXH_PUBLIC_API unsigned XXH_versionNumber (void); - - -/*-********************************************************************** -* 32-bits hash -************************************************************************/ -typedef unsigned int XXH32_hash_t; - -/*! XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". - The memory between input & input+length must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */ -XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); - -/*====== Streaming ======*/ -typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); - -XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); - -/* -These functions generate the xxHash of an input provided in multiple segments. -Note that, for small input, they are slower than single-call functions, due to state management. -For small input, prefer `XXH32()` and `XXH64()` . - -XXH state must first be allocated, using XXH*_createState() . - -Start a new hash by initializing state with a seed, using XXH*_reset(). - -Then, feed the hash state by calling XXH*_update() as many times as necessary. -Obviously, input must be allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. - -Finally, a hash value can be produced anytime, by using XXH*_digest(). -This function returns the nn-bits hash as an int or long long. - -It's still possible to continue inserting input into the hash state after a digest, -and generate some new hashes later on, by calling again XXH*_digest(). - -When done, free XXH state space if it was allocated dynamically. -*/ - -/*====== Canonical representation ======*/ - -typedef struct { unsigned char digest[4]; } XXH32_canonical_t; -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); - -/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ - - -#ifndef XXH_NO_LONG_LONG -/*-********************************************************************** -* 64-bits hash -************************************************************************/ -typedef unsigned long long XXH64_hash_t; - -/*! XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". - "seed" can be used to alter the result predictably. - This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark). -*/ -XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); - -/*====== Streaming ======*/ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); -XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); - -XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); - -/*====== Canonical representation ======*/ -typedef struct { unsigned char digest[8]; } XXH64_canonical_t; -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); -#endif /* XXH_NO_LONG_LONG */ - - -#ifdef XXH_STATIC_LINKING_ONLY - -/* ================================================================================================ - This section contains definitions which are not guaranteed to remain stable. - They may change in future versions, becoming incompatible with a different version of the library. - They shall only be used with static linking. - Never use these definitions in association with dynamic linking ! -=================================================================================================== */ - -/* These definitions are only meant to make possible - static allocation of XXH state, on stack or in a struct for example. - Never use members directly. */ - -struct XXH32_state_s { - unsigned total_len_32; - unsigned large_len; - unsigned v1; - unsigned v2; - unsigned v3; - unsigned v4; - unsigned mem32[4]; /* buffer defined as U32 for alignment */ - unsigned memsize; - unsigned reserved; /* never read nor write, will be removed in a future version */ -}; /* typedef'd to XXH32_state_t */ - -#ifndef XXH_NO_LONG_LONG /* remove 64-bits support */ -struct XXH64_state_s { - unsigned long long total_len; - unsigned long long v1; - unsigned long long v2; - unsigned long long v3; - unsigned long long v4; - unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ - unsigned memsize; - unsigned reserved[2]; /* never read nor write, will be removed in a future version */ -}; /* typedef'd to XXH64_state_t */ -#endif - -#ifdef XXH_PRIVATE_API -# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ -#endif - -#endif /* XXH_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* XXHASH_H_5627135585666179 */ diff --git a/contrib/lizard/programs/.gitignore b/contrib/lizard/programs/.gitignore deleted file mode 100644 index c2bfcafa9ee..00000000000 --- a/contrib/lizard/programs/.gitignore +++ /dev/null @@ -1,15 +0,0 @@ -# local binary (Makefile) -lizard -lizardc32 -datagen -frametest -frametest32 -fullbench -fullbench32 -fuzzer -fuzzer32 -*.exe - -# tests files -tmp* -_* \ No newline at end of file diff --git a/contrib/lizard/programs/COPYING b/contrib/lizard/programs/COPYING deleted file mode 100644 index d159169d105..00000000000 --- a/contrib/lizard/programs/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/contrib/lizard/programs/README.md b/contrib/lizard/programs/README.md deleted file mode 100644 index ee116c116c9..00000000000 --- a/contrib/lizard/programs/README.md +++ /dev/null @@ -1,73 +0,0 @@ -Command Line Interface for LZ4 library -============================================ - -Command Line Interface (CLI) can be created using the `make` command without any additional parameters. -There are also multiple targets that create different variations of CLI: -- `lizard` : default CLI, with a command line syntax close to gzip -- `lizardc32` : Same as `lizard`, but forced to compile in 32-bits mode - - -#### Aggregation of parameters -CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`. - - - -#### Benchmark in Command Line Interface -CLI includes in-memory compression benchmark module for lizard. -The benchmark is conducted using a given filename. -The file is read into memory. -It makes benchmark more precise as it eliminates I/O overhead. - -The benchmark measures ratio, compressed size, compression and decompression speed. -One can select compression levels starting from `-b` and ending with `-e`. -The `-i` parameter selects a number of seconds used for each of tested levels. - - - -#### Usage of Command Line Interface -The full list of commands can be obtained with `-h` or `-H` parameter: -``` -Usage : - lizard [arg] [input] [output] - -input : a filename - with no FILE, or when FILE is - or stdin, read standard input -Arguments : - -10...-19 : compression method fastLZ4 = 16-bit bytewise codewords - higher number == more compression but slower - -20...-29 : compression method LIZv1 = 24-bit bytewise codewords - -30...-39 : compression method fastLZ4 + Huffman - -40...-49 : compression method LIZv1 + Huffman - -d : decompression (default for .liz extension) - -z : force compression - -f : overwrite output without prompting ---rm : remove source file(s) after successful de/compression - -h/-H : display help/long help and exit - -Advanced arguments : - -V : display Version number and exit - -v : verbose mode - -q : suppress warnings; specify twice to suppress errors too - -c : force write to standard output, even if it is the console - -t : test compressed file integrity - -m : multiple input files (implies automatic output filenames) - -r : operate recursively on directories (sets also -m) - -l : compress using Legacy format (Linux kernel compression) - -B# : Block size [1-7] = 128KB, 256KB, 1MB, 4MB, 16MB, 64MB, 256MB (default : 4) - -BD : Block dependency (improve compression ratio) ---no-frame-crc : disable stream checksum (default:enabled) ---content-size : compressed frame includes original size (default:not present) ---[no-]sparse : sparse mode (default:enabled on file, disabled on stdout) -Benchmark arguments : - -b# : benchmark file(s), using # compression level (default : 1) - -e# : test all compression levels from -bX to # (default : 1) - -i# : minimum evaluation time in seconds (default : 3s) - -B# : cut file into independent blocks of size # bytes [32+] - or predefined block size [1-7] (default: 4) -``` - -#### License - -All files in this directory are licensed under GPL-v2. -See [COPYING](COPYING) for details. -The text of the license is also included at the top of each source file. diff --git a/contrib/lizard/programs/bench.c b/contrib/lizard/programs/bench.c deleted file mode 100644 index 3fe56d1c060..00000000000 --- a/contrib/lizard/programs/bench.c +++ /dev/null @@ -1,502 +0,0 @@ -/* - bench.c - Demo program to benchmark open-source compression algorithms - Copyright (C) Yann Collet 2012-2016 - Copyright (C) Przemyslaw Skibinski 2016-2017 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ - - -/*-************************************ -* Compiler options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif -#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) -# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ -#endif - - -/* ************************************* -* Includes -***************************************/ -#include "platform.h" /* Compiler options */ -#include /* malloc, free */ -#include /* memset */ -#include /* fprintf, fopen, ftello64 */ -#include /* clock_t, clock, CLOCKS_PER_SEC */ - -#include "datagen.h" /* RDG_genBuffer */ -#include "xxhash/xxhash.h" -#include "lizard_common.h" -#include "lizard_decompress.h" -#define UTIL_WITHOUT_BASIC_TYPES -#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */ - -#define Lizard_isError(errcode) (errcode==0) - - -/* ************************************* -* Constants -***************************************/ -#ifndef LIZARD_GIT_COMMIT_STRING -# define LIZARD_GIT_COMMIT_STRING "" -#else -# define LIZARD_GIT_COMMIT_STRING LIZARD_EXPAND_AND_QUOTE(Lizard_GIT_COMMIT) -#endif - -#define NBSECONDS 3 -#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ -#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ -#define COOLPERIOD_SEC 10 -#define DECOMP_MULT 2 /* test decompression DECOMP_MULT times longer than compression */ - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); - -static U32 g_compressibilityDefault = 50; - - -/* ************************************* -* console display -***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ - -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ - { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } -static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; -static clock_t g_time = 0; - - -/* ************************************* -* Exceptions -***************************************/ -#ifndef DEBUG -# define DEBUG 0 -#endif -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ - exit(error); \ -} - - -/* ************************************* -* Benchmark Parameters -***************************************/ -static U32 g_nbSeconds = NBSECONDS; -static size_t g_blockSize = 0; -int g_additionalParam = 0; - -void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } - -void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } - -void BMK_SetNbSeconds(unsigned nbSeconds) -{ - g_nbSeconds = nbSeconds; - DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds); -} - -void BMK_SetBlockSize(size_t blockSize) -{ - g_blockSize = blockSize; -} - - -/* ******************************************************** -* Bench functions -**********************************************************/ -typedef struct { - const char* srcPtr; - size_t srcSize; - char* cPtr; - size_t cRoom; - size_t cSize; - char* resPtr; - size_t resSize; -} blockParam_t; - - - -static int BMK_benchMem(const void* srcBuffer, size_t srcSize, - const char* displayName, int cLevel, - const size_t* fileSizes, U32 nbFiles) -{ - size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; - blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); - size_t const maxCompressedSize = Lizard_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ - void* const compressedBuffer = malloc(maxCompressedSize); - void* const resultBuffer = malloc(srcSize); - U32 nbBlocks; - UTIL_time_t ticksPerSecond; - - /* checks */ - if (!compressedBuffer || !resultBuffer || !blockTable) - EXM_THROW(31, "allocation error : not enough memory"); - - /* init */ - if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ - UTIL_initTimer(&ticksPerSecond); - - /* Init blockTable data */ - { const char* srcPtr = (const char*)srcBuffer; - char* cPtr = (char*)compressedBuffer; - char* resPtr = (char*)resultBuffer; - U32 fileNb; - for (nbBlocks=0, fileNb=0; fileNb ACTIVEPERIOD_MICROSEC) { - DISPLAYLEVEL(2, "\rcooling down ... \r"); - UTIL_sleep(COOLPERIOD_SEC); - UTIL_getTime(&coolTime); - } - - /* Compression */ - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); - if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ - - UTIL_sleepMilli(1); /* give processor time to other processes */ - UTIL_waitForNextTick(ticksPerSecond); - UTIL_getTime(&clockStart); - - if (!cCompleted) { /* still some time to do compression tests */ - U32 nbLoops = 0; - do { - U32 blockNb; - for (blockNb=0; blockNbmaxTime; - } } - - cSize = 0; - { U32 blockNb; for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, - (double)srcSize / fastestC ); - - (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ -#if 1 - /* Decompression */ - if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ - - UTIL_sleepMilli(1); /* give processor time to other processes */ - UTIL_waitForNextTick(ticksPerSecond); - UTIL_getTime(&clockStart); - - if (!dCompleted) { - U32 nbLoops = 0; - do { - U32 blockNb; - for (blockNb=0; blockNb(DECOMP_MULT*maxTime); - } } - - markNb = (markNb+1) % NB_MARKS; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", - marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio, - (double)srcSize / fastestC, - (double)srcSize / fastestD ); - - /* CRC Checking */ - { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); - if (crcOrig!=crcCheck) { - size_t u; - DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); - for (u=0; u u) break; - bacc += blockTable[segNb].srcSize; - } - pos = (U32)(u - bacc); - bNb = pos / (128 KB); - DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos); - break; - } - if (u==srcSize-1) { /* should never happen */ - DISPLAY("no difference detected\n"); - } } - break; - } } /* CRC Checking */ -#endif - } /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */ - - if (g_displayLevel == 1) { - double cSpeed = (double)srcSize / fastestC; - double dSpeed = (double)srcSize / fastestD; - if (g_additionalParam) - DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam); - else - DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); - } - DISPLAYLEVEL(2, "%2i#\n", cLevel); - } /* Bench */ - - /* clean up */ - free(blockTable); - free(compressedBuffer); - free(resultBuffer); - return 0; -} - - -static size_t BMK_findMaxMem(U64 requiredMem) -{ - size_t step = 64 MB; - BYTE* testmem=NULL; - - requiredMem = (((requiredMem >> 26) + 1) << 26); - requiredMem += 2*step; - if (requiredMem > maxMemory) requiredMem = maxMemory; - - while (!testmem) { - if (requiredMem > step) requiredMem -= step; - else requiredMem >>= 1; - testmem = (BYTE*) malloc ((size_t)requiredMem); - } - free (testmem); - - /* keep some space available */ - if (requiredMem > step) requiredMem -= step; - else requiredMem >>= 1; - - return (size_t)requiredMem; -} - - -static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, - const char* displayName, int cLevel, int cLevelLast, - const size_t* fileSizes, unsigned nbFiles) -{ - int l; - - const char* pch = strrchr(displayName, '\\'); /* Windows */ - if (!pch) pch = strrchr(displayName, '/'); /* Linux */ - if (pch) displayName = pch+1; - - SET_REALTIME_PRIORITY; - - if (g_displayLevel == 1 && !g_additionalParam) - DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", LIZARD_VERSION_STRING, LIZARD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); - - if (cLevelLast < cLevel) cLevelLast = cLevel; - - for (l=cLevel; l <= cLevelLast; l++) { - BMK_benchMem(srcBuffer, benchedSize, - displayName, l, - fileSizes, nbFiles); - } -} - - -/*! BMK_loadFiles() : - Loads `buffer` with content of files listed within `fileNamesTable`. - At most, fills `buffer` entirely */ -static void BMK_loadFiles(void* buffer, size_t bufferSize, - size_t* fileSizes, - const char** fileNamesTable, unsigned nbFiles) -{ - size_t pos = 0, totalSize = 0; - unsigned n; - for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ - { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); - DISPLAYLEVEL(4, "readSize=%llu\n", (unsigned long long)readSize); - pos += readSize; } - fileSizes[n] = (size_t)fileSize; - totalSize += (size_t)fileSize; - fclose(f); - } - - if (totalSize == 0) EXM_THROW(12, "no data to bench"); -} - -static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, - int cLevel, int cLevelLast) -{ - void* srcBuffer; - size_t benchedSize; - size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); - U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); - char mfName[20] = {0}; - - if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); - - /* Memory allocation & restrictions */ - benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; - if (benchedSize==0) EXM_THROW(12, "not enough memory"); - if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; - if (benchedSize > LIZARD_MAX_INPUT_SIZE) { - benchedSize = LIZARD_MAX_INPUT_SIZE; - DISPLAY("File(s) bigger than Lizard's max input size; testing %u MB only...\n", (U32)(benchedSize >> 20)); - } - else - if (benchedSize < totalSizeToLoad) { - DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); - } - srcBuffer = malloc(benchedSize + !benchedSize); /* avoid alloc of zero */ - if (!srcBuffer) EXM_THROW(12, "not enough memory"); - - /* Load input buffer */ - BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); - - /* Bench */ - snprintf (mfName, sizeof(mfName), " %u files", nbFiles); - { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; - BMK_benchCLevel(srcBuffer, benchedSize, - displayName, cLevel, cLevelLast, - fileSizes, nbFiles); - } - - /* clean up */ - free(srcBuffer); - free(fileSizes); -} - - -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) -{ - char name[20] = {0}; - size_t benchedSize = 10000000; - void* const srcBuffer = malloc(benchedSize); - - /* Memory allocation */ - if (!srcBuffer) EXM_THROW(21, "not enough memory"); - - /* Fill input buffer */ - RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); - - /* Bench */ - snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1); - - /* clean up */ - free(srcBuffer); -} - - -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - int cLevel, int cLevelLast) -{ - double const compressibility = (double)g_compressibilityDefault / 100; - - if (cLevel < LIZARD_MIN_CLEVEL) cLevel = LIZARD_MIN_CLEVEL; - if (cLevel > LIZARD_MAX_CLEVEL) cLevel = LIZARD_MAX_CLEVEL; - if (cLevelLast > LIZARD_MAX_CLEVEL) cLevelLast = LIZARD_MAX_CLEVEL; - if (cLevelLast < cLevel) cLevelLast = cLevel; - if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); - - if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility); - else - BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast); - return 0; -} diff --git a/contrib/lizard/programs/bench.h b/contrib/lizard/programs/bench.h deleted file mode 100644 index feff51293f3..00000000000 --- a/contrib/lizard/programs/bench.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - bench.h - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2016 - Copyright (C) Przemyslaw Skibinski 2016-2017 - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ -#ifndef BENCH_H_12 -#define BENCH_H_12 - -#include - -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - int cLevel, int cLevelLast); - -/* Set Parameters */ -void BMK_SetNbSeconds(unsigned nbLoops); -void BMK_SetBlockSize(size_t blockSize); -void BMK_setAdditionalParam(int additionalParam); -void BMK_setNotificationLevel(unsigned level); - -#endif /* BENCH_H_125623623633 */ diff --git a/contrib/lizard/programs/datagen.c b/contrib/lizard/programs/datagen.c deleted file mode 100644 index 87d6044830a..00000000000 --- a/contrib/lizard/programs/datagen.c +++ /dev/null @@ -1,188 +0,0 @@ -/* - datagen.c - compressible data generator test tool - Copyright (C) Yann Collet 2012-2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ - -/************************************** -* Includes -**************************************/ -#include "platform.h" /* Compiler options, SET_BINARY_MODE */ -#include "util.h" /* U32 */ -#include /* malloc */ -#include /* FILE, fwrite */ -#include /* memcpy */ - - -/************************************** -* Constants -**************************************/ -#define KB *(1 <<10) - -#define PRIME1 2654435761U -#define PRIME2 2246822519U - - -/************************************** -* Local types -**************************************/ -#define LTLOG 13 -#define LTSIZE (1<> (32 - r))) -static unsigned int RDG_rand(U32* src) -{ - U32 rand32 = *src; - rand32 *= PRIME1; - rand32 ^= PRIME2; - rand32 = RDG_rotl32(rand32, 13); - *src = rand32; - return rand32; -} - - -static void RDG_fillLiteralDistrib(litDistribTable lt, double ld) -{ - BYTE const firstChar = ld <= 0.0 ? 0 : '('; - BYTE const lastChar = ld <= 0.0 ? 255 : '}'; - BYTE character = ld <= 0.0 ? 0 : '0'; - U32 u = 0; - - while (u lastChar) character = firstChar; - } -} - - -static BYTE RDG_genChar(U32* seed, const litDistribTable lt) -{ - U32 id = RDG_rand(seed) & LTMASK; - return (lt[id]); -} - - -#define RDG_DICTSIZE (32 KB) -#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767) -#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15) -void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr) -{ - BYTE* buffPtr = (BYTE*)buffer; - const U32 matchProba32 = (U32)(32768 * matchProba); - size_t pos = prefixSize; - U32* seed = seedPtr; - - /* special case */ - while (matchProba >= 1.0) - { - size_t size0 = RDG_rand(seed) & 3; - size0 = (size_t)1 << (16 + size0 * 2); - size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/ - if (buffSize < pos + size0) - { - memset(buffPtr+pos, 0, buffSize-pos); - return; - } - memset(buffPtr+pos, 0, size0); - pos += size0; - buffPtr[pos-1] = RDG_genChar(seed, lt); - } - - /* init */ - if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1; - - /* Generate compressible data */ - while (pos < buffSize) - { - /* Select : Literal (char) or Match (within 32K) */ - if (RDG_RAND15BITS < matchProba32) - { - /* Copy (within 32K) */ - size_t match; - size_t d; - int length = RDG_RANDLENGTH + 4; - U32 offset = RDG_RAND15BITS + 1; - if (offset > pos) offset = (U32)pos; - match = pos - offset; - d = pos + length; - if (d > buffSize) d = buffSize; - while (pos < d) buffPtr[pos++] = buffPtr[match++]; - } - else - { - /* Literal (noise) */ - size_t d; - size_t length = RDG_RANDLENGTH; - d = pos + length; - if (d > buffSize) d = buffSize; - while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt); - } - } -} - - -void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) -{ - litDistribTable lt; - if (litProba==0.0) litProba = matchProba / 4.5; - RDG_fillLiteralDistrib(lt, litProba); - RDG_genBlock(buffer, size, 0, matchProba, lt, &seed); -} - - -#define RDG_BLOCKSIZE (128 KB) -void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed) -{ - BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE]; - U64 total = 0; - size_t genBlockSize = RDG_BLOCKSIZE; - litDistribTable lt; - - /* init */ - if (litProba==0.0) litProba = matchProba / 4.5; - RDG_fillLiteralDistrib(lt, litProba); - SET_BINARY_MODE(stdout); - - /* Generate dict */ - RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, lt, &seed); - - /* Generate compressible data */ - while (total < size) - { - RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, lt, &seed); - if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total); - total += genBlockSize; - fwrite(buff, 1, genBlockSize, stdout); - /* update dict */ - memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE); - } -} diff --git a/contrib/lizard/programs/datagen.h b/contrib/lizard/programs/datagen.h deleted file mode 100644 index 2cc41d2a3ff..00000000000 --- a/contrib/lizard/programs/datagen.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - datagen.h - compressible data generator header - Copyright (C) Yann Collet 2012-2015 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ - - -#include /* size_t */ - -void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed); -void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed); -/* RDG_genOut - Generate 'size' bytes of compressible data into stdout. - Compressibility can be controlled using 'matchProba'. - 'LitProba' is optional, and affect variability of bytes. If litProba==0.0, default value is used. - Generated data can be selected using 'seed'. - If (matchProba, litProba and seed) are equal, the function always generate the same content. - - RDG_genBuffer - Same as RDG_genOut, but generate data into provided buffer -*/ diff --git a/contrib/lizard/programs/lizard.1 b/contrib/lizard/programs/lizard.1 deleted file mode 100644 index b5d84213442..00000000000 --- a/contrib/lizard/programs/lizard.1 +++ /dev/null @@ -1,229 +0,0 @@ -\." -\." lizard.1: This is a manual page for 'lizard' program. This file is part of the -\." lizard project. -\." Author: Yann Collet -\." -. -\." No hyphenation -.hy 0 -.nr HY 0 -. -.TH lizard "1" "2015-03-21" "lizard" "User Commands" -.SH NAME -\fBlizard, unlizard, lizardcat\fR \- Compress or decompress .liz files - -.SH SYNOPSIS -.TP 5 -\fBlizard\fR [\fBOPTIONS\fR] [-|INPUT-FILE] -.PP -.B unlizard -is equivalent to -.BR "lizard \-d" -.br -.B lizardcat -is equivalent to -.BR "lizard \-dc" -.br -.PP -When writing scripts that need to decompress files, -it is recommended to always use the name -.B lizard -with appropriate arguments -.RB ( "lizard \-d" -or -.BR "lizard \-dc" ) -instead of the names -.B unlizard -and -.BR lizardcat . - - -.SH DESCRIPTION -.PP -\fBlizard\fR is an extremely fast lossless compression algorithm, -based on \fBbyte-aligned LZ77\fR family of compression scheme. -\fBlizard\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi-core CPUs. -It features an extremely fast decoder, with speed in multiple GB/s per core, -typically reaching RAM speed limit on multi-core systems. -The native file format is the -.B .liz -format. - -.B lizard -supports a command line syntax similar but not identical to -.BR gzip (1). -Differences are : -\fBlizard\fR preserve original files ; -\fBlizard file1 file2\fR means : compress file1 \fIinto\fR file2 ; -\fBlizard file\fR shows real-time statistics during compression . - -Default behaviors can be modified by opt-in commands, described below. -\fBlizard --quiet --multiple\fR more closely mimics \fBgzip\fR behavior. - -.SS "Concatenation of .liz files" -It is possible to concatenate -.B .liz -files as is. -.B lizard -will decompress such files as if they were a single -.B .liz -file. For example: - lizard file1 > foo.liz - lizard file2 >> foo.liz - then - lizardcat foo.liz - is equivalent to : - cat file1 file2 - -.PP - -.SH OPTIONS -. -.SS "Short commands concatenation" -In some cases, some options can be expressed using short command -.B "-x" -or long command -.B "--long-word" . -Short commands can be concatenated together. For example, -.B "-d -c" -is equivalent to -.B "-dc" . -Long commands cannot be concatenated. -They must be clearly separated by a space. -.SS "Multiple commands" -When multiple contradictory commands are issued on a same command line, -only the latest one will be applied. -. -.SS "Operation mode" -.TP -.BR \-z ", " \-\-compress -Compress. -This is the default operation mode -when no operation mode option is specified , -no other operation mode is implied from the command name -(for example, -.B unlizard -implies -.B \-\-decompress ), -nor from the input file name -(for example, a file extension -.B .liz -implies -.B \-\-decompress -by default). -.B -z -can also be used to force compression of an already compressed -.B .liz -file. -.TP -.BR \-d ", " \-\-decompress ", " \-\-uncompress -Decompress. -.B --decompress -is also the default operation when the input filename has an -.B .liz -extensionq -.TP -.BR \-t ", " \-\-test -Test the integrity of compressed -.B .liz -files. -The decompressed data is discarded. -No files are created nor removed. -. -.SS "Operation modifiers" -.TP -.B \-1 - fast compression (default) -.TP -.B \-9 - high compression - -.TP -.BR \-f ", " --[no-]force - This option has several effects: -.RS -.IP \(bu 3 -If the target file already exists, -overwrite it without prompting. -.IP \(bu 3 -When used with -.B \-\-decompress -and -.B lizard -cannot recognize the type of the source file, -copy the source file as is to standard output. -This allows -.B lizardcat -.B \-\-force -to be used like -.BR cat (1) -for files that have not been compressed with -.BR lizard . -.RE - -.TP -.BR \-c ", " \--stdout ", " \--to-stdout - force write to standard output, even if it is the console - -.TP -.BR \-m ", " \--multiple - Multiple file names. - By default, the second filename is used as the destination filename for the compressed file. - With -.B -m -, you can specify any number of input filenames. Each of them will be compressed -independently, and the resulting name of each compressed file will be -.B filename.liz -. - -.TP -.B \-B# - block size [4-7](default : 7) - B1 = 128KB, B2=256KB, B3=1MB, B4=4MB, B5=16MB, B6=64MB, B7=256MB -.TP -.B \-BD - block dependency (improves compression ratio on small blocks) -.TP -.B \--[no-]frame-crc - select frame checksum (default:enabled) -.TP -.B \--[no-]content-size - header includes original size (default:not present) - Note : this option can only be activated when the original size can be determined, -hence for a file. It won't work with unknown source size, such as stdin or pipe. -.TP -.B \--[no-]sparse - sparse mode support (default:enabled on file, disabled on stdout) -.TP -.B \-l - use Legacy format (useful for Linux Kernel compression) -. -.SS "Other options" -.TP -.BR \-v ", " --verbose - verbose mode -.TP -.BR \-q ", " --quiet - suppress warnings and real-time statistics; specify twice to suppress errors too -.TP -.B \-h/\-H - display help/long help and exit -.TP -.BR \-V ", " \--version - display Version number and exit -.TP -.BR \-k ", " \--keep - Don't delete source file. -This is default behavior anyway, so this option is just for compatibility with gzip/xz. -.TP -.B \-b - benchmark file(s) -.TP -.B \-i# - iteration loops [1-9](default : 3), benchmark mode only - -.SH BUGS -Report bugs at: https://github.com/inikep/lizard/issues - -.SH AUTHOR -Yann Collet diff --git a/contrib/lizard/programs/lizardcli.c b/contrib/lizard/programs/lizardcli.c deleted file mode 100644 index 2a63fe8e001..00000000000 --- a/contrib/lizard/programs/lizardcli.c +++ /dev/null @@ -1,581 +0,0 @@ -/* - Lizardcli - Lizard Command Line Interface - Copyright (C) Yann Collet 2011-2016 - Copyright (C) Przemyslaw Skibinski 2016-2017 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ -/* - Note : this is stand-alone program. - It is not part of Lizard compression library, it is a user program of the Lizard library. - The license of Lizard library is BSD. - The license of xxHash library is BSD. - The license of this compression CLI program is GPLv2. -*/ - - -/**************************** -* Includes -*****************************/ -#include "platform.h" /* Compiler options, IS_CONSOLE */ -#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ -#include /* fprintf, getchar */ -#include /* exit, calloc, free */ -#include /* strcmp, strlen */ -#include "bench.h" /* BMK_benchFile, BMK_SetNbIterations, BMK_SetBlocksize, BMK_SetPause */ -#include "lizardio.h" /* LizardIO_compressFilename, LizardIO_decompressFilename, LizardIO_compressMultipleFilenames */ -#include "lizard_compress.h" /* LizardHC_DEFAULT_CLEVEL, LIZARD_VERSION_STRING */ - - - -/***************************** -* Constants -******************************/ -#define COMPRESSOR_NAME "Lizard command line interface" -#define AUTHOR "Y.Collet & P.Skibinski" -#define WELCOME_MESSAGE "%s %i-bit %s by %s (%s)\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LIZARD_VERSION_STRING, AUTHOR, __DATE__ -#define LIZARD_EXTENSION ".liz" -#define LIZARDCAT "lizardcat" -#define UNLIZARD "unlizard" - -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - -#define LIZARD_BLOCKSIZEID_DEFAULT 4 - - -/*-************************************ -* Macros -***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */ - - -/*-************************************ -* Exceptions -***************************************/ -#define DEBUG 0 -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ - exit(error); \ -} - - -/*-************************************ -* Version modifiers -***************************************/ -#define EXTENDED_ARGUMENTS -#define EXTENDED_HELP -#define EXTENDED_FORMAT -#define DEFAULT_COMPRESSOR LizardIO_compressFilename -#define DEFAULT_DECOMPRESSOR LizardIO_decompressFilename -int LizardIO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel); /* hidden function */ - - -/*-*************************** -* Functions -*****************************/ -static int usage(const char* exeName) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] [input] [output]\n", exeName); - DISPLAY( "\n"); - DISPLAY( "input : a filename\n"); - DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); - DISPLAY( "Arguments :\n"); - DISPLAY( " -10...-19 : compression method fastLZ4 = 16-bit bytewise codewords\n"); - DISPLAY( " higher number == more compression but slower\n"); - DISPLAY( " -20...-29 : compression method LIZv1 = 24-bit bytewise codewords\n"); -#ifndef LIZARD_NO_HUFFMAN - DISPLAY( " -30...-39 : compression method fastLZ4 + Huffman\n"); - DISPLAY( " -40...-49 : compression method LIZv1 + Huffman\n"); -#endif - DISPLAY( " -d : decompression (default for %s extension)\n", LIZARD_EXTENSION); - DISPLAY( " -z : force compression\n"); - DISPLAY( " -f : overwrite output without prompting \n"); - DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); - DISPLAY( " -h/-H : display help/long help and exit\n"); - return 0; -} - -static int usage_advanced(const char* exeName) -{ - DISPLAY(WELCOME_MESSAGE); - usage(exeName); - DISPLAY( "\n"); - DISPLAY( "Advanced arguments :\n"); - DISPLAY( " -V : display Version number and exit\n"); - DISPLAY( " -v : verbose mode\n"); - DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); - DISPLAY( " -c : force write to standard output, even if it is the console\n"); - DISPLAY( " -t : test compressed file integrity\n"); - DISPLAY( " -m : multiple input files (implies automatic output filenames)\n"); -#ifdef UTIL_HAS_CREATEFILELIST - DISPLAY( " -r : operate recursively on directories (sets also -m)\n"); -#endif - DISPLAY( " -B# : Block size [1-7] = 128KB, 256KB, 1MB, 4MB, 16MB, 64MB, 256MB (default : 4)\n"); - DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); - /* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */ - DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n"); - DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); - DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); - DISPLAY( "Benchmark arguments :\n"); - DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n"); - DISPLAY( " -e# : test all compression levels from -bX to # (default : 1)\n"); - DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); - DISPLAY( " -B# : cut file into independent blocks of size # bytes [32+]\n"); - DISPLAY( " or predefined block size [1-7] (default: 4)\n"); - EXTENDED_HELP; - return 0; -} - -static int usage_longhelp(const char* exeName) -{ - usage_advanced(exeName); - DISPLAY( "\n"); - DISPLAY( "****************************\n"); - DISPLAY( "***** Advanced comment *****\n"); - DISPLAY( "****************************\n"); - DISPLAY( "\n"); - DISPLAY( "Which values can [output] have ? \n"); - DISPLAY( "---------------------------------\n"); - DISPLAY( "[output] : a filename \n"); - DISPLAY( " '%s', or '-' for standard output (pipe mode)\n", stdoutmark); - DISPLAY( " '%s' to discard output (test mode) \n", NULL_OUTPUT); - DISPLAY( "[output] can be left empty. In this case, it receives the following value :\n"); - DISPLAY( " - if stdout is not the console, then [output] = stdout \n"); - DISPLAY( " - if stdout is console : \n"); - DISPLAY( " + for compression, output to filename%s \n", LIZARD_EXTENSION); - DISPLAY( " + for decompression, output to filename without '%s'\n", LIZARD_EXTENSION); - DISPLAY( " > if input filename has no '%s' extension : error \n", LIZARD_EXTENSION); - DISPLAY( "\n"); - DISPLAY( "stdin, stdout and the console : \n"); - DISPLAY( "--------------------------------\n"); - DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n"); - DISPLAY( "%s will refuse to read from console, or write to console \n", exeName); - DISPLAY( "except if '-c' command is specified, to force output to console \n"); - DISPLAY( "\n"); - DISPLAY( "Simple example :\n"); - DISPLAY( "----------------\n"); - DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.liz'\n"); - DISPLAY( " %s filename\n", exeName); - DISPLAY( "\n"); - DISPLAY( "Short arguments can be aggregated. For example :\n"); - DISPLAY( "----------------------------------\n"); - DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n"); - DISPLAY( " %s -9 -f filename \n", exeName); - DISPLAY( " is equivalent to :\n"); - DISPLAY( " %s -9f filename \n", exeName); - DISPLAY( "\n"); - DISPLAY( "%s can be used in 'pure pipe mode'. For example :\n", exeName); - DISPLAY( "-------------------------------------\n"); - DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n"); - DISPLAY( " generator | %s | consumer \n", exeName); - return 0; -} - -static int badusage(const char* exeName) -{ - DISPLAYLEVEL(1, "Incorrect parameters\n"); - if (displayLevel >= 1) usage(exeName); - exit(1); -} - - -static void waitEnter(void) -{ - DISPLAY("Press enter to continue...\n"); - (void)getchar(); -} - - -static const char* lastNameFromPath(const char* path) -{ - const char* name = strrchr(path, '/'); - if (name==NULL) name = strrchr(path, '\\'); /* windows */ - if (name==NULL) return path; - return name+1; -} - - -/*! readU32FromChar() : - @return : unsigned integer value reach from input in `char` format - Will also modify `*stringPtr`, advancing it to position where it stopped reading. - Note : this function can overflow if result > MAX_UINT */ -static unsigned readU32FromChar(const char** stringPtr) -{ - unsigned result = 0; - while ((**stringPtr >='0') && (**stringPtr <='9')) - result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; - return result; -} - -typedef enum { om_auto, om_compress, om_decompress, om_test, om_bench } operationMode_e; - -int main(int argc, const char** argv) -{ - int i, - cLevel=1, - cLevelLast=1, - forceStdout=0, - main_pause=0, - multiple_inputs=0, - operationResult=0; - operationMode_e mode = om_auto; - const char* input_filename = NULL; - const char* output_filename= NULL; - char* dynNameSpace = NULL; - const char** inFileNames = (const char**) calloc(argc, sizeof(char*)); - unsigned ifnIdx=0; - const char nullOutput[] = NULL_OUTPUT; - const char extension[] = LIZARD_EXTENSION; - size_t blockSize = LizardIO_setBlockSizeID(LIZARD_BLOCKSIZEID_DEFAULT); - const char* const exeName = lastNameFromPath(argv[0]); -#ifdef UTIL_HAS_CREATEFILELIST - const char** extendedFileList = NULL; - char* fileNamesBuf = NULL; - unsigned fileNamesNb, recursive=0; -#endif - - /* Init */ - if (inFileNames==NULL) { - DISPLAY("Allocation error : not enough memory \n"); - return 1; - } - inFileNames[0] = stdinmark; - LizardIO_setOverwrite(0); - - /* lizardcat predefined behavior */ - if (!strcmp(exeName, LIZARDCAT)) { - mode = om_decompress; - LizardIO_setOverwrite(1); - forceStdout=1; - output_filename=stdoutmark; - displayLevel=1; - multiple_inputs=1; - } - if (!strcmp(exeName, UNLIZARD)) { mode = om_decompress; } - - /* command switches */ - for(i=1; i='0') && (*argument<='9')) { - cLevel = readU32FromChar(&argument); - argument--; - continue; - } - - - switch(argument[0]) - { - /* Display help */ - case 'V': DISPLAY(WELCOME_MESSAGE); goto _cleanup; /* Version */ - case 'h': usage_advanced(exeName); goto _cleanup; - case 'H': usage_longhelp(exeName); goto _cleanup; - - case 'e': - argument++; - cLevelLast = readU32FromChar(&argument); - argument--; - break; - - /* Compression (default) */ - case 'z': mode = om_compress; break; - - /* Decoding */ - case 'd': mode = om_decompress; break; - - /* Force stdout, even if stdout==console */ - case 'c': forceStdout=1; output_filename=stdoutmark; break; - - /* Test integrity */ - case 't': mode = om_test; break; - - /* Overwrite */ - case 'f': LizardIO_setOverwrite(1); break; - - /* Verbose mode */ - case 'v': displayLevel++; break; - - /* Quiet mode */ - case 'q': if (displayLevel) displayLevel--; break; - - /* keep source file (default anyway, so useless) (for xz/lzma compatibility) */ - case 'k': LizardIO_setRemoveSrcFile(0); break; - - /* Modify Block Properties */ - case 'B': - while (argument[1]!=0) { - int exitBlockProperties=0; - switch(argument[1]) - { - case 'D': LizardIO_setBlockMode(LizardIO_blockLinked); argument++; break; - case 'X': LizardIO_setBlockChecksumMode(1); argument ++; break; /* disabled by default */ - default : - if (argument[1] < '0' || argument[1] > '9') { - exitBlockProperties=1; - break; - } else { - unsigned B; - argument++; - B = readU32FromChar(&argument); - argument--; - if (B < 1) badusage(exeName); - if (B <= 7) { - blockSize = LizardIO_setBlockSizeID(B); - BMK_SetBlockSize(blockSize); - DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10)); - } else { - if (B < 32) badusage(exeName); - BMK_SetBlockSize(B); - if (B >= 1024) { - DISPLAYLEVEL(2, "bench: using blocks of size %u KB \n", (U32)(B>>10)); - } else { - DISPLAYLEVEL(2, "bench: using blocks of size %u bytes \n", (U32)(B)); - } - } - break; - } - } - if (exitBlockProperties) break; - } - break; - - /* Benchmark */ - case 'b': mode = om_bench; multiple_inputs=1; - break; - -#ifdef UTIL_HAS_CREATEFILELIST - /* recursive */ - case 'r': recursive=1; - /* fallthrough */ -#endif - /* Treat non-option args as input files. See https://code.google.com/p/lizard/issues/detail?id=151 */ - /* fallthrough */ - case 'm': multiple_inputs=1; - break; - - /* Modify Nb Seconds (benchmark only) */ - case 'i': - { unsigned iters; - argument++; - iters = readU32FromChar(&argument); - argument--; - BMK_setNotificationLevel(displayLevel); - BMK_SetNbSeconds(iters); /* notification if displayLevel >= 3 */ - } - break; - - /* Pause at the end (hidden option) */ - case 'p': main_pause=1; break; - - /* Specific commands for customized versions */ - EXTENDED_ARGUMENTS; - - /* Unrecognised command */ - default : badusage(exeName); - } - } - continue; - } - - /* Store in *inFileNames[] if -m is used. */ - if (multiple_inputs) { inFileNames[ifnIdx++]=argument; continue; } - - /* Store first non-option arg in input_filename to preserve original cli logic. */ - if (!input_filename) { input_filename=argument; continue; } - - /* Second non-option arg in output_filename to preserve original cli logic. */ - if (!output_filename) { - output_filename=argument; - if (!strcmp (output_filename, nullOutput)) output_filename = nulmark; - continue; - } - - /* 3rd non-option arg should not exist */ - DISPLAYLEVEL(1, "Warning : %s won't be used ! Do you want multiple input files (-m) ? \n", argument); - } - - DISPLAYLEVEL(3, WELCOME_MESSAGE); - if ((mode == om_compress) || (mode == om_bench)) DISPLAYLEVEL(4, "Blocks size : %i KB\n", (U32)(blockSize>>10)); - - if (multiple_inputs) { - input_filename = inFileNames[0]; -#ifdef UTIL_HAS_CREATEFILELIST - if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ - extendedFileList = UTIL_createFileList(inFileNames, ifnIdx, &fileNamesBuf, &fileNamesNb); - if (extendedFileList) { - unsigned u; - for (u=0; u try to select one automatically (when possible) */ - while ((!output_filename) && (multiple_inputs==0)) { - if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } /* Default to stdout whenever possible (i.e. not a console) */ - if (mode == om_auto) { /* auto-determine compression or decompression, based on file extension */ - size_t const inSize = strlen(input_filename); - size_t const extSize = strlen(LIZARD_EXTENSION); - size_t const extStart= (inSize > extSize) ? inSize-extSize : 0; - if (!strcmp(input_filename+extStart, LIZARD_EXTENSION)) mode = om_decompress; - else mode = om_compress; - } - if (mode == om_compress) { /* compression to file */ - size_t const l = strlen(input_filename); - dynNameSpace = (char*)calloc(1,l+5); - if (dynNameSpace==NULL) { perror(exeName); exit(1); } - strcpy(dynNameSpace, input_filename); - strcat(dynNameSpace, LIZARD_EXTENSION); - output_filename = dynNameSpace; - DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename); - break; - } - if (mode == om_decompress) {/* decompression to file (automatic name will work only if input filename has correct format extension) */ - size_t outl; - size_t const inl = strlen(input_filename); - dynNameSpace = (char*)calloc(1,inl+1); - if (dynNameSpace==NULL) { perror(exeName); exit(1); } - strcpy(dynNameSpace, input_filename); - outl = inl; - if (inl>4) - while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) dynNameSpace[outl--]=0; - if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(exeName); } - output_filename = dynNameSpace; - DISPLAYLEVEL(2, "Decoding file %s \n", output_filename); - } - break; - } - - /* Check if output is defined as console; trigger an error in this case */ - if (!output_filename) output_filename = "*\\dummy^!//"; - if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) { - DISPLAYLEVEL(1, "refusing to write to console without -c\n"); - exit(1); - } - - /* Downgrade notification level in stdout and multiple file mode */ - if (!strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; - if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; - - /* IO Stream/File */ - LizardIO_setNotificationLevel(displayLevel); - if (ifnIdx == 0) multiple_inputs = 0; - if (mode == om_decompress) { - if (multiple_inputs) - operationResult = LizardIO_decompressMultipleFilenames(inFileNames, ifnIdx, !strcmp(output_filename,stdoutmark) ? stdoutmark : LIZARD_EXTENSION); - else - operationResult = LizardIO_decompressFilename(input_filename, output_filename); - } else { /* compression is default action */ - { - if (multiple_inputs) - operationResult = LizardIO_compressMultipleFilenames(inFileNames, ifnIdx, LIZARD_EXTENSION, cLevel); - else - operationResult = LizardIO_compressFilename(input_filename, output_filename, cLevel); - } - } - -_cleanup: - if (main_pause) waitEnter(); - if (dynNameSpace) free(dynNameSpace); -#ifdef UTIL_HAS_CREATEFILELIST - if (extendedFileList) - UTIL_freeFileList(extendedFileList, fileNamesBuf); - else -#endif - free((void*)inFileNames); - return operationResult; -} diff --git a/contrib/lizard/programs/lizardio.c b/contrib/lizard/programs/lizardio.c deleted file mode 100644 index 7638a55178a..00000000000 --- a/contrib/lizard/programs/lizardio.c +++ /dev/null @@ -1,896 +0,0 @@ -/* - Lizardio.c - Lizard File/Stream Interface - Copyright (C) Yann Collet 2011-2015 - Copyright (C) Przemyslaw Skibinski 2016-2017 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ -/* - Note : this is stand-alone program. - It is not part of Lizard compression library, it is a user code of the Lizard library. - - The license of Lizard library is BSD. - - The license of xxHash library is BSD. - - The license of this source file is GPLv2. -*/ - -/*-************************************ -* Compiler options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif -#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) -# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ -#endif - - -/***************************** -* Includes -*****************************/ -#include "platform.h" /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE, PLATFORM_POSIX_VERSION, __64BIT__ */ -#include "util.h" /* UTIL_getFileStat, UTIL_setFileStat */ -#include /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ -#include /* malloc, free */ -#include /* strcmp, strlen */ -#include /* clock */ -#include /* stat64 */ -#include /* stat64 */ -#include "lizardio.h" -#include "lizard_frame.h" - - - -/***************************** -* Constants -*****************************/ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define _1BIT 0x01 -#define _2BITS 0x03 -#define _3BITS 0x07 -#define _4BITS 0x0F -#define _8BITS 0xFF - -#define MAGICNUMBER_SIZE 4 -#define LIZARDIO_MAGICNUMBER 0x184D2206U -#define LIZARDIO_SKIPPABLE0 0x184D2A50U -#define LIZARDIO_SKIPPABLEMASK 0xFFFFFFF0U - -#define CACHELINE 64 -#define MIN_STREAM_BUFSIZE (192 KB) -#define LIZARDIO_BLOCKSIZEID_DEFAULT 7 - -#define sizeT sizeof(size_t) -#define maskT (sizeT - 1) - - -/************************************** -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static int g_displayLevel = 0; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ - -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if (((clock_t)(g_time - clock()) > refreshRate) || (g_displayLevel>=4)) \ - { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stderr); } } -static const clock_t refreshRate = CLOCKS_PER_SEC / 6; -static clock_t g_time = 0; - - -/************************************** -* Local Parameters -**************************************/ -static int g_overwrite = 1; -static int g_testMode = 0; -static int g_blockSizeId = LIZARDIO_BLOCKSIZEID_DEFAULT; -static int g_blockChecksum = 0; -static int g_streamChecksum = 1; -static int g_blockIndependence = 1; -static int g_sparseFileSupport = 1; -static int g_contentSizeFlag = 0; - - -/************************************** -* Exceptions -***************************************/ -#ifndef DEBUG -# define DEBUG 0 -#endif -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, " \n"); \ - exit(error); \ -} - - -/************************************** -* Version modifiers -**************************************/ -#define EXTENDED_ARGUMENTS -#define EXTENDED_HELP -#define EXTENDED_FORMAT -#define DEFAULT_DECOMPRESSOR LizardIO_decompressLizardF - - -/* ************************************************** */ -/* ****************** Parameters ******************** */ -/* ************************************************** */ - -/* Default setting : overwrite = 1; return : overwrite mode (0/1) */ -int LizardIO_setOverwrite(int yes) -{ - g_overwrite = (yes!=0); - return g_overwrite; -} - -/* Default setting : testMode = 0; return : testMode (0/1) */ -int LizardIO_setTestMode(int yes) -{ - g_testMode = (yes!=0); - return g_testMode; -} - -/* blockSizeID : valid values : 1-7 */ -size_t LizardIO_setBlockSizeID(unsigned bsid) -{ - static const int blockSizeTable[] = { 128 KB, 256 KB, 1 MB, 4 MB, 16 MB, 64 MB, 256 MB }; - static const unsigned minBlockSizeID = 1; - static const unsigned maxBlockSizeID = 7; - if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return 0; - - g_blockSizeId = bsid; - return blockSizeTable[g_blockSizeId-minBlockSizeID]; -} - - -static size_t LizardIO_GetBlockSize_FromBlockId(unsigned blockSizeID) -{ - static const size_t blockSizes[7] = { 128 KB, 256 KB, 1 MB, 4 MB, 16 MB, 64 MB, 256 MB }; - - if (blockSizeID == 0) blockSizeID = LIZARDIO_BLOCKSIZEID_DEFAULT; - blockSizeID -= 1; - if (blockSizeID >= 7) blockSizeID = LIZARDIO_BLOCKSIZEID_DEFAULT - 1; - - return blockSizes[blockSizeID]; -} - - - -int LizardIO_setBlockMode(LizardIO_blockMode_t blockMode) -{ - g_blockIndependence = (blockMode == LizardIO_blockIndependent); - return g_blockIndependence; -} - -/* Default setting : no checksum */ -int LizardIO_setBlockChecksumMode(int xxhash) -{ - g_blockChecksum = (xxhash != 0); - return g_blockChecksum; -} - -/* Default setting : checksum enabled */ -int LizardIO_setStreamChecksumMode(int xxhash) -{ - g_streamChecksum = (xxhash != 0); - return g_streamChecksum; -} - -/* Default setting : 0 (no notification) */ -int LizardIO_setNotificationLevel(int level) -{ - g_displayLevel = level; - return g_displayLevel; -} - -/* Default setting : 0 (disabled) */ -int LizardIO_setSparseFile(int enable) -{ - g_sparseFileSupport = (enable!=0); - return g_sparseFileSupport; -} - -/* Default setting : 0 (disabled) */ -int LizardIO_setContentSize(int enable) -{ - g_contentSizeFlag = (enable!=0); - return g_contentSizeFlag; -} - -static U32 g_removeSrcFile = 0; -void LizardIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } - - - -/* ************************************************************************ ** -** ********************** Lizard File / Pipe compression ********************* ** -** ************************************************************************ */ - -static int LizardIO_isSkippableMagicNumber(unsigned int magic) { return (magic & LIZARDIO_SKIPPABLEMASK) == LIZARDIO_SKIPPABLE0; } - - -/** LizardIO_openSrcFile() : - * condition : `dstFileName` must be non-NULL. - * @result : FILE* to `dstFileName`, or NULL if it fails */ -static FILE* LizardIO_openSrcFile(const char* srcFileName) -{ - FILE* f; - - if (!strcmp (srcFileName, stdinmark)) { - DISPLAYLEVEL(4,"Using stdin for input\n"); - f = stdin; - SET_BINARY_MODE(stdin); - } else { - f = fopen(srcFileName, "rb"); - if ( f==NULL ) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno)); - } - - return f; -} - -/** FIO_openDstFile() : - * condition : `dstFileName` must be non-NULL. - * @result : FILE* to `dstFileName`, or NULL if it fails */ -static FILE* LizardIO_openDstFile(const char* dstFileName) -{ - FILE* f; - - if (!strcmp (dstFileName, stdoutmark)) { - DISPLAYLEVEL(4,"Using stdout for output\n"); - f = stdout; - SET_BINARY_MODE(stdout); - if (g_sparseFileSupport==1) { - g_sparseFileSupport = 0; - DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); - } - } else { - if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */ - f = fopen( dstFileName, "rb" ); - if (f != NULL) { /* dest exists, prompt for overwrite authorization */ - fclose(f); - if (g_displayLevel <= 1) { /* No interaction possible */ - DISPLAY("%s already exists; not overwritten \n", dstFileName); - return NULL; - } - DISPLAY("%s already exists; do you wish to overwrite (y/N) ? ", dstFileName); - { int ch = getchar(); - if ((ch!='Y') && (ch!='y')) { - DISPLAY(" not overwritten \n"); - return NULL; - } - while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */ - } } } - f = fopen( dstFileName, "wb" ); - if (f==NULL) DISPLAYLEVEL(1, "%s: %s\n", dstFileName, strerror(errno)); - } - - /* sparse file */ - if (f && g_sparseFileSupport) { SET_SPARSE_FILE_MODE(f); } - - return f; -} - - -/* unoptimized version; solves endianess & alignment issues */ -static void LizardIO_writeLE32 (void* p, unsigned value32) -{ - unsigned char* dstPtr = (unsigned char*)p; - dstPtr[0] = (unsigned char)value32; - dstPtr[1] = (unsigned char)(value32 >> 8); - dstPtr[2] = (unsigned char)(value32 >> 16); - dstPtr[3] = (unsigned char)(value32 >> 24); -} - - - -/********************************************* -* Compression using Frame format -*********************************************/ - -typedef struct { - void* srcBuffer; - size_t srcBufferSize; - void* dstBuffer; - size_t dstBufferSize; - LizardF_compressionContext_t ctx; -} cRess_t; - -static cRess_t LizardIO_createCResources(void) -{ - const size_t blockSize = (size_t)LizardIO_GetBlockSize_FromBlockId (g_blockSizeId); - cRess_t ress; - - LizardF_errorCode_t const errorCode = LizardF_createCompressionContext(&(ress.ctx), LIZARDF_VERSION); - if (LizardF_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LizardF context : %s", LizardF_getErrorName(errorCode)); - - /* Allocate Memory */ - ress.srcBuffer = malloc(blockSize); - ress.srcBufferSize = blockSize; - ress.dstBufferSize = LizardF_compressFrameBound(blockSize, NULL); /* cover worst case */ - ress.dstBuffer = malloc(ress.dstBufferSize); - if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); - - return ress; -} - -static void LizardIO_freeCResources(cRess_t ress) -{ - free(ress.srcBuffer); - free(ress.dstBuffer); - { LizardF_errorCode_t const errorCode = LizardF_freeCompressionContext(ress.ctx); - if (LizardF_isError(errorCode)) EXM_THROW(38, "Error : can't free LizardF context resource : %s", LizardF_getErrorName(errorCode)); } -} - -/* - * LizardIO_compressFilename_extRess() - * result : 0 : compression completed correctly - * 1 : missing or pb opening srcFileName - */ -static int LizardIO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) -{ - unsigned long long filesize = 0; - unsigned long long compressedfilesize = 0; - FILE* srcFile; - FILE* dstFile; - void* const srcBuffer = ress.srcBuffer; - void* const dstBuffer = ress.dstBuffer; - const size_t dstBufferSize = ress.dstBufferSize; - const size_t blockSize = (size_t)LizardIO_GetBlockSize_FromBlockId (g_blockSizeId); - size_t readSize; - LizardF_compressionContext_t ctx = ress.ctx; /* just a pointer */ - LizardF_preferences_t prefs; - - /* Init */ - srcFile = LizardIO_openSrcFile(srcFileName); - if (srcFile == NULL) return 1; - dstFile = LizardIO_openDstFile(dstFileName); - if (dstFile == NULL) { fclose(srcFile); return 1; } - memset(&prefs, 0, sizeof(prefs)); - - - /* Set compression parameters */ - prefs.autoFlush = 1; - prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = (LizardF_blockMode_t)g_blockIndependence; - prefs.frameInfo.blockSizeID = (LizardF_blockSizeID_t)g_blockSizeId; - prefs.frameInfo.contentChecksumFlag = (LizardF_contentChecksum_t)g_streamChecksum; - if (g_contentSizeFlag) { - U64 const fileSize = UTIL_getFileSize(srcFileName); - prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ - if (fileSize==0) - DISPLAYLEVEL(3, "Warning : cannot determine input content size \n"); - } - - /* read first block */ - readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile); - if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName); - filesize += readSize; - - /* single-block file */ - if (readSize < blockSize) { - /* Compress in single pass */ - size_t const cSize = LizardF_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs); - if (LizardF_isError(cSize)) EXM_THROW(31, "Compression failed : %s", LizardF_getErrorName(cSize)); - compressedfilesize = cSize; - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", - (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */ - - /* Write Block */ - { size_t const sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile); - if (sizeCheck!=cSize) EXM_THROW(32, "Write error : cannot write compressed block"); - } } - - else - - /* multiple-blocks file */ - { - /* Write Archive Header */ - size_t headerSize = LizardF_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs); - if (LizardF_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LizardF_getErrorName(headerSize)); - { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(34, "Write error : cannot write header"); } - compressedfilesize += headerSize; - - /* Main Loop */ - while (readSize>0) { - size_t outSize; - - /* Compress Block */ - outSize = LizardF_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); - if (LizardF_isError(outSize)) EXM_THROW(35, "Compression failed : %s", LizardF_getErrorName(outSize)); - compressedfilesize += outSize; - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); - - /* Write Block */ - { size_t const sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); - if (sizeCheck!=outSize) EXM_THROW(36, "Write error : cannot write compressed block"); } - - /* Read next block */ - readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); - filesize += readSize; - } - if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); - - /* End of Stream mark */ - headerSize = LizardF_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); - if (LizardF_isError(headerSize)) EXM_THROW(38, "End of file generation failed : %s", LizardF_getErrorName(headerSize)); - - { size_t const sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); - if (sizeCheck!=headerSize) EXM_THROW(39, "Write error : cannot write end of stream"); } - compressedfilesize += headerSize; - } - - /* Release files */ - fclose (srcFile); - fclose (dstFile); - - /* Copy owner, file permissions and modification time */ - { stat_t statbuf; - if (strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) && UTIL_getFileStat(srcFileName, &statbuf)) - UTIL_setFileStat(dstFileName, &statbuf); - } - - if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ - - /* Final Status */ - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - filesize, compressedfilesize, (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */ - - return 0; -} - - -int LizardIO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel) -{ - clock_t const start = clock(); - cRess_t const ress = LizardIO_createCResources(); - - int const issueWithSrcFile = LizardIO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); - - /* Free resources */ - LizardIO_freeCResources(ress); - - /* Final Status */ - { clock_t const end = clock(); - double const seconds = (double)(end - start) / CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); - } - - return issueWithSrcFile; -} - - -#define FNSPACE 30 -int LizardIO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) -{ - int i; - int missed_files = 0; - char* dstFileName = (char*)malloc(FNSPACE); - size_t ofnSize = FNSPACE; - const size_t suffixSize = strlen(suffix); - cRess_t const ress = LizardIO_createCResources(); - - if (dstFileName == NULL) return ifntSize; /* not enough memory */ - - /* loop on each file */ - for (i=0; i 1 GB) { - int const seekResult = UTIL_fseek(file, 1 GB, SEEK_CUR); - if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); - storedSkips -= 1 GB; - } - - while (ptrT < bufferTEnd) { - size_t seg0SizeT = segmentSizeT; - size_t nb0T; - - /* count leading zeros */ - if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; - bufferSizeT -= seg0SizeT; - for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; - storedSkips += (unsigned)(nb0T * sizeT); - - if (nb0T != seg0SizeT) { /* not all 0s */ - int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); - storedSkips = 0; - seg0SizeT -= nb0T; - ptrT += nb0T; - { size_t const sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file); - if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block"); - } } - ptrT += seg0SizeT; - } - - if (bufferSize & maskT) { /* size not multiple of sizeT : implies end of block */ - const char* const restStart = (const char*)bufferTEnd; - const char* restPtr = restStart; - size_t const restSize = bufferSize & maskT; - const char* const restEnd = restStart + restSize; - for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; - storedSkips += (unsigned) (restPtr - restStart); - if (restPtr != restEnd) { - int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); - storedSkips = 0; - { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); - if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block"); - } } - } - - return storedSkips; -} - -static void LizardIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) -{ - if (storedSkips>0) { /* implies g_sparseFileSupport>0 */ - int const seekResult = UTIL_fseek(file, storedSkips-1, SEEK_CUR); - if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); - { const char lastZeroByte[1] = { 0 }; - size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); - if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); - } } -} - - - -typedef struct { - void* srcBuffer; - size_t srcBufferSize; - void* dstBuffer; - size_t dstBufferSize; - FILE* dstFile; - LizardF_decompressionContext_t dCtx; -} dRess_t; - -static const size_t LizardIO_dBufferSize = 64 KB; -static unsigned g_magicRead = 0; -static dRess_t LizardIO_createDResources(void) -{ - dRess_t ress; - - /* init */ - LizardF_errorCode_t const errorCode = LizardF_createDecompressionContext(&ress.dCtx, LIZARDF_VERSION); - if (LizardF_isError(errorCode)) EXM_THROW(60, "Can't create LizardF context : %s", LizardF_getErrorName(errorCode)); - - /* Allocate Memory */ - ress.srcBufferSize = LizardIO_dBufferSize; - ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = LizardIO_dBufferSize; - ress.dstBuffer = malloc(ress.dstBufferSize); - if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); - - ress.dstFile = NULL; - return ress; -} - -static void LizardIO_freeDResources(dRess_t ress) -{ - LizardF_errorCode_t errorCode = LizardF_freeDecompressionContext(ress.dCtx); - if (LizardF_isError(errorCode)) EXM_THROW(69, "Error : can't free LizardF context resource : %s", LizardF_getErrorName(errorCode)); - free(ress.srcBuffer); - free(ress.dstBuffer); -} - - -static unsigned long long LizardIO_decompressLizardF(dRess_t ress, FILE* srcFile, FILE* dstFile) -{ - unsigned long long filesize = 0; - LizardF_errorCode_t nextToLoad; - unsigned storedSkips = 0; - - /* Init feed with magic number (already consumed from FILE* sFile) */ - { size_t inSize = MAGICNUMBER_SIZE; - size_t outSize= 0; - LizardIO_writeLE32(ress.srcBuffer, LIZARDIO_MAGICNUMBER); - nextToLoad = LizardF_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL); - if (LizardF_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LizardF_getErrorName(nextToLoad)); - } - - /* Main Loop */ - for (;nextToLoad;) { - size_t readSize; - size_t pos = 0; - size_t decodedBytes = ress.dstBufferSize; - - /* Read input */ - if (nextToLoad > ress.srcBufferSize) nextToLoad = ress.srcBufferSize; - readSize = fread(ress.srcBuffer, 1, nextToLoad, srcFile); - if (!readSize) break; /* reached end of file or stream */ - - while ((pos < readSize) || (decodedBytes == ress.dstBufferSize)) { /* still to read, or still to flush */ - /* Decode Input (at least partially) */ - size_t remaining = readSize - pos; - decodedBytes = ress.dstBufferSize; - nextToLoad = LizardF_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); - if (LizardF_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LizardF_getErrorName(nextToLoad)); - pos += remaining; - - /* Write Block */ - if (decodedBytes) { - if (!g_testMode) - storedSkips = LizardIO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips); - filesize += decodedBytes; - DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); - } - - if (!nextToLoad) break; - } - } - /* can be out because readSize == 0, which could be an fread() error */ - if (ferror(srcFile)) EXM_THROW(67, "Read error"); - - if (!g_testMode) LizardIO_fwriteSparseEnd(dstFile, storedSkips); - if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream"); - - return filesize; -} - - -#define PTSIZE (64 KB) -#define PTSIZET (PTSIZE / sizeof(size_t)) -static unsigned long long LizardIO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) -{ - size_t buffer[PTSIZET]; - size_t readBytes = 1; - unsigned long long total = MAGICNUMBER_SIZE; - unsigned storedSkips = 0; - - size_t const sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error"); - - while (readBytes) { - readBytes = fread(buffer, 1, PTSIZE, finput); - total += readBytes; - storedSkips = LizardIO_fwriteSparse(foutput, buffer, readBytes, storedSkips); - } - if (ferror(finput)) EXM_THROW(51, "Read Error") - - LizardIO_fwriteSparseEnd(foutput, storedSkips); - return total; -} - - -/** Safely handle cases when (unsigned)offset > LONG_MAX */ -static int fseek_u32(FILE *fp, unsigned offset, int where) -{ - const unsigned stepMax = 1U << 30; - int errorNb = 0; - - if (where != SEEK_CUR) return -1; /* Only allows SEEK_CUR */ - while (offset > 0) { - unsigned s = offset; - if (s > stepMax) s = stepMax; - errorNb = UTIL_fseek(fp, (long) s, SEEK_CUR); - if (errorNb != 0) break; - offset -= s; - } - return errorNb; -} - -#define ENDOFSTREAM ((unsigned long long)-1) -static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput) -{ - unsigned char MNstore[MAGICNUMBER_SIZE]; - unsigned magicNumber; - static unsigned nbCalls = 0; - - /* init */ - nbCalls++; - - /* Check Archive Header */ - if (g_magicRead) { /* magic number already read from finput (see legacy frame)*/ - magicNumber = g_magicRead; - g_magicRead = 0; - } else { - size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput); - if (nbReadBytes==0) { nbCalls = 0; return ENDOFSTREAM; } /* EOF */ - if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); - magicNumber = LizardIO_readLE32(MNstore); /* Little Endian format */ - } - if (LizardIO_isSkippableMagicNumber(magicNumber)) magicNumber = LIZARDIO_SKIPPABLE0; /* fold skippable magic numbers */ - - switch(magicNumber) - { - case LIZARDIO_MAGICNUMBER: - return LizardIO_decompressLizardF(ress, finput, foutput); - case LIZARDIO_SKIPPABLE0: - DISPLAYLEVEL(4, "Skipping detected skippable area \n"); - { size_t const nbReadBytes = fread(MNstore, 1, 4, finput); - if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); } - { unsigned const size = LizardIO_readLE32(MNstore); /* Little Endian format */ - int const errorNb = fseek_u32(finput, size, SEEK_CUR); - if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); } - return 0; - EXTENDED_FORMAT; /* macro extension for custom formats */ - default: - if (nbCalls == 1) { /* just started */ - if (!g_testMode && g_overwrite) { - nbCalls = 0; - return LizardIO_passThrough(finput, foutput, MNstore); - } - EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */ - } - DISPLAYLEVEL(2, "Stream followed by undecodable data\n"); - return ENDOFSTREAM; - } -} - - -static int LizardIO_decompressSrcFile(dRess_t ress, const char* input_filename, const char* output_filename) -{ - FILE* const foutput = ress.dstFile; - unsigned long long filesize = 0, decodedSize=0; - FILE* finput; - - /* Init */ - finput = LizardIO_openSrcFile(input_filename); - if (finput==NULL) return 1; - - /* Loop over multiple streams */ - do { - decodedSize = selectDecoder(ress, finput, foutput); - if (decodedSize != ENDOFSTREAM) - filesize += decodedSize; - } while (decodedSize != ENDOFSTREAM); - - /* Close */ - fclose(finput); - - if (g_removeSrcFile) { if (remove(input_filename)) EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno)); } /* remove source file : --rm */ - - /* Final Status */ - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "%-20.20s : decoded %llu bytes \n", input_filename, filesize); - (void)output_filename; - - return 0; -} - - -static int LizardIO_decompressDstFile(dRess_t ress, const char* input_filename, const char* output_filename) -{ - FILE* foutput; - - /* Init */ - foutput = LizardIO_openDstFile(output_filename); - if (foutput==NULL) return 1; /* failure */ - - ress.dstFile = foutput; - LizardIO_decompressSrcFile(ress, input_filename, output_filename); - - fclose(foutput); - - /* Copy owner, file permissions and modification time */ - { stat_t statbuf; - if (strcmp (input_filename, stdinmark) && strcmp (output_filename, stdoutmark) && UTIL_getFileStat(input_filename, &statbuf)) - UTIL_setFileStat(output_filename, &statbuf); - } - - return 0; -} - - -int LizardIO_decompressFilename(const char* input_filename, const char* output_filename) -{ - dRess_t const ress = LizardIO_createDResources(); - clock_t const start = clock(); - - int const missingFiles = LizardIO_decompressDstFile(ress, input_filename, output_filename); - - { clock_t const end = clock(); - double const seconds = (double)(end - start) / CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds); - } - - LizardIO_freeDResources(ress); - return missingFiles; -} - - -int LizardIO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix) -{ - int i; - int skippedFiles = 0; - int missingFiles = 0; - char* outFileName = (char*)malloc(FNSPACE); - size_t ofnSize = FNSPACE; - size_t const suffixSize = strlen(suffix); - dRess_t ress = LizardIO_createDResources(); - - if (outFileName==NULL) return ifntSize; /* not enough memory */ - ress.dstFile = LizardIO_openDstFile(stdoutmark); - - for (i=0; i /* size_t */ - - -/* ************************************************** */ -/* Special input/output values */ -/* ************************************************** */ -#define NULL_OUTPUT "null" -static const char stdinmark[] = "stdin"; -static const char stdoutmark[] = "stdout"; -#ifdef _WIN32 -static const char nulmark[] = "nul"; -#else -static const char nulmark[] = "/dev/null"; -#endif - - -/* ************************************************** */ -/* ****************** Functions ********************* */ -/* ************************************************** */ - -int LizardIO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel); -int LizardIO_decompressFilename(const char* input_filename, const char* output_filename); - -int LizardIO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel); -int LizardIO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix); - - -/* ************************************************** */ -/* ****************** Parameters ******************** */ -/* ************************************************** */ - -/* Default setting : overwrite = 1; - return : overwrite mode (0/1) */ -int LizardIO_setOverwrite(int yes); - -/* Default setting : testMode = 0; - return : testMode (0/1) */ -int LizardIO_setTestMode(int yes); - -/* blockSizeID : valid values : 4-5-6-7 - return : 0 if error, blockSize if OK */ -size_t LizardIO_setBlockSizeID(unsigned blockSizeID); - -/* Default setting : independent blocks */ -typedef enum { LizardIO_blockLinked=0, LizardIO_blockIndependent} LizardIO_blockMode_t; -int LizardIO_setBlockMode(LizardIO_blockMode_t blockMode); - -/* Default setting : no block checksum */ -int LizardIO_setBlockChecksumMode(int xxhash); - -/* Default setting : stream checksum enabled */ -int LizardIO_setStreamChecksumMode(int xxhash); - -/* Default setting : 0 (no notification) */ -int LizardIO_setNotificationLevel(int level); - -/* Default setting : 0 (disabled) */ -int LizardIO_setSparseFile(int enable); - -/* Default setting : 0 (disabled) */ -int LizardIO_setContentSize(int enable); - -void LizardIO_setRemoveSrcFile(unsigned flag); - - -#endif /* LIZARDIO_H_237902873 */ diff --git a/contrib/lizard/programs/platform.h b/contrib/lizard/programs/platform.h deleted file mode 100644 index 51ce1ac2d14..00000000000 --- a/contrib/lizard/programs/platform.h +++ /dev/null @@ -1,145 +0,0 @@ -/* - platform.h - compiler and OS detection - Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -*/ - -#ifndef PLATFORM_H_MODULE -#define PLATFORM_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/* ************************************** -* Compiler Options -****************************************/ -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before and */ -# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ -# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ -# endif -#endif - - -/* ************************************** -* Detect 64-bit OS -* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros -****************************************/ -#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ - || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ - || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ - || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ - || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ - || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ - || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ - || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ -# if !defined(__64BIT__) -# define __64BIT__ 1 -# endif -#endif - - -/* ********************************************************* -* Turn on Large Files support (>4GB) for 32-bit Linux/Unix -***********************************************************/ -#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */ -# if !defined(_FILE_OFFSET_BITS) -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# endif -# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ -# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ -# endif -# if defined(_AIX) || defined(__hpux) -# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ -# endif -#endif - - -/* ************************************************************ -* Detect POSIX version -* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows -* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX -* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION -***************************************************************/ -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ - || defined(__midipix__) || defined(__VMS)) -# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \ - || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */ -# define PLATFORM_POSIX_VERSION 200112L -# else -# if defined(__linux__) || defined(__linux) -# ifndef _POSIX_C_SOURCE -# define _POSIX_C_SOURCE 200112L /* use feature test macro */ -# endif -# endif -# include /* declares _POSIX_VERSION */ -# if defined(_POSIX_VERSION) /* POSIX compliant */ -# define PLATFORM_POSIX_VERSION _POSIX_VERSION -# else -# define PLATFORM_POSIX_VERSION 0 -# endif -# endif -#endif -#if !defined(PLATFORM_POSIX_VERSION) -# define PLATFORM_POSIX_VERSION -1 -#endif - - -/*-********************************************* -* Detect if isatty() and fileno() are available -************************************************/ -#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) -# include /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _isatty */ -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#else -# define IS_CONSOLE(stdStream) 0 -#endif - - -/****************************** -* OS-specific Includes -******************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) -# include /* _O_BINARY */ -# include /* _setmode, _fileno, _get_osfhandle */ -# if !defined(__DJGPP__) -# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ -# include /* FSCTL_SET_SPARSE */ -# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } -# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } -# else -# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) -# define SET_SPARSE_FILE_MODE(file) -# endif -#else -# define SET_BINARY_MODE(file) -# define SET_SPARSE_FILE_MODE(file) -#endif - - - -#if defined (__cplusplus) -} -#endif - -#endif /* PLATFORM_H_MODULE */ diff --git a/contrib/lizard/programs/util.h b/contrib/lizard/programs/util.h deleted file mode 100644 index 4c2313eef00..00000000000 --- a/contrib/lizard/programs/util.h +++ /dev/null @@ -1,497 +0,0 @@ -/* - util.h - utility functions - Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -*/ - -#ifndef UTIL_H_MODULE -#define UTIL_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/*-**************************************** -* Dependencies -******************************************/ -#include "platform.h" /* PLATFORM_POSIX_VERSION */ -#include /* malloc */ -#include /* size_t, ptrdiff_t */ -#include /* fprintf */ -#include /* strlen, strncpy */ -#include /* stat, utime */ -#include /* stat */ -#if defined(_MSC_VER) -# include /* utime */ -# include /* _chmod */ -#else -# include /* chown, stat */ -# include /* utime */ -#endif -#include /* time */ -#include - - - -#ifndef UTIL_WITHOUT_BASIC_TYPES -/*-************************************************************** -* Basic Types -*****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; -#endif -#endif - - -/* ************************************************************ -* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW -***************************************************************/ -#if defined(_MSC_VER) && (_MSC_VER >= 1400) -# define UTIL_fseek _fseeki64 -#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ -# define UTIL_fseek fseeko -#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) -# define UTIL_fseek fseeko64 -#else -# define UTIL_fseek fseek -#endif - - -/*-**************************************** -* Sleep functions: Windows - Posix - others -******************************************/ -#if defined(_WIN32) -# include -# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) -# define UTIL_sleep(s) Sleep(1000*s) -# define UTIL_sleepMilli(milli) Sleep(milli) -#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ -# include -# include /* setpriority */ -# include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ -# if defined(PRIO_PROCESS) -# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20) -# else -# define SET_REALTIME_PRIORITY /* disabled */ -# endif -# define UTIL_sleep(s) sleep(s) -# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ -# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } -# else -# define UTIL_sleepMilli(milli) /* disabled */ -# endif -#else -# define SET_REALTIME_PRIORITY /* disabled */ -# define UTIL_sleep(s) /* disabled */ -# define UTIL_sleepMilli(milli) /* disabled */ -#endif - - -/* ************************************* -* Constants -***************************************/ -#define LIST_SIZE_INCREASE (8*1024) - - -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */ -#endif -#if defined(__GNUC__) -# define UTIL_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define UTIL_STATIC static inline -#elif defined(_MSC_VER) -# define UTIL_STATIC static __inline -#else -# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - - -/*-**************************************** -* Time functions -******************************************/ -#if (PLATFORM_POSIX_VERSION >= 1) -#include -#include /* times */ - typedef U64 UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=sysconf(_SC_CLK_TCK); } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { struct tms junk; clock_t newTicks = (clock_t) times(&junk); (void)junk; *x = (UTIL_time_t)newTicks; } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / ticksPerSecond; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / ticksPerSecond; } -#elif defined(_WIN32) /* Windows */ - typedef LARGE_INTEGER UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } -#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */ - typedef clock_t UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } -#endif - - -/* returns time span in microseconds */ -UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond ) -{ - UTIL_time_t clockEnd; - UTIL_getTime(&clockEnd); - return UTIL_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd); -} - - -UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) -{ - UTIL_time_t clockStart, clockEnd; - UTIL_getTime(&clockStart); - do { - UTIL_getTime(&clockEnd); - } while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0); -} - - - -/*-**************************************** -* File functions -******************************************/ -#if defined(_MSC_VER) - #define chmod _chmod - typedef struct __stat64 stat_t; -#else - typedef struct stat stat_t; -#endif - - -UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) -{ - int res = 0; - struct utimbuf timebuf; - - timebuf.actime = time(NULL); - timebuf.modtime = statbuf->st_mtime; - res += utime(filename, &timebuf); /* set access and modification times */ - -#if !defined(_WIN32) - res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */ -#endif - - res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */ - - errno = 0; - return -res; /* number of errors is returned */ -} - - -UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) -{ - int r; -#if defined(_MSC_VER) - r = _stat64(infilename, statbuf); - if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ -#else - r = stat(infilename, statbuf); - if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */ -#endif - return 1; -} - - -UTIL_STATIC int UTIL_isRegFile(const char* infilename) -{ - stat_t statbuf; - return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */ -} - - -UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) -{ - int r; - stat_t statbuf; -#if defined(_MSC_VER) - r = _stat64(infilename, &statbuf); - if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; -#else - r = stat(infilename, &statbuf); - if (!r && S_ISDIR(statbuf.st_mode)) return 1; -#endif - return 0; -} - - -UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct __stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ -#elif defined(__MINGW32__) && defined (__MSVCRT__) - struct _stati64 statbuf; - r = _stati64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ -#else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ -#endif - return (U64)statbuf.st_size; -} - - -UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) -{ - U64 total = 0; - unsigned n; - for (n=0; n= *bufEnd) { - ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; - *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); - *bufEnd = *bufStart + newListSize; - if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } - } - if (*bufStart + *pos + pathLength < *bufEnd) { - strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos)); - *pos += pathLength + 1; - nbFiles++; - } - } - free(path); - } while (FindNextFileA(hFile, &cFile)); - - FindClose(hFile); - return nbFiles; -} - -#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ -# define UTIL_HAS_CREATEFILELIST -# include /* opendir, readdir */ -# include /* strerror, memcpy */ - -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) -{ - DIR *dir; - struct dirent *entry; - char* path; - int dirLength, fnameLength, pathLength, nbFiles = 0; - - if (!(dir = opendir(dirName))) { - fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); - return 0; - } - - dirLength = (int)strlen(dirName); - errno = 0; - while ((entry = readdir(dir)) != NULL) { - if (strcmp (entry->d_name, "..") == 0 || - strcmp (entry->d_name, ".") == 0) continue; - fnameLength = (int)strlen(entry->d_name); - path = (char*) malloc(dirLength + fnameLength + 2); - if (!path) { closedir(dir); return 0; } - memcpy(path, dirName, dirLength); - path[dirLength] = '/'; - memcpy(path+dirLength+1, entry->d_name, fnameLength); - pathLength = dirLength+1+fnameLength; - path[pathLength] = 0; - - if (UTIL_isDirectory(path)) { - nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */ - if (*bufStart == NULL) { free(path); closedir(dir); return 0; } - } else { - if (*bufStart + *pos + pathLength >= *bufEnd) { - ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; - *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); - *bufEnd = *bufStart + newListSize; - if (*bufStart == NULL) { free(path); closedir(dir); return 0; } - } - if (*bufStart + *pos + pathLength < *bufEnd) { - strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos)); - *pos += pathLength + 1; - nbFiles++; - } - } - free(path); - errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */ - } - - if (errno != 0) { - fprintf(stderr, "readdir(%s) error: %s\n", dirName, strerror(errno)); - free(*bufStart); - *bufStart = NULL; - } - closedir(dir); - return nbFiles; -} - -#else - -UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) -{ - (void)bufStart; (void)bufEnd; (void)pos; - fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); - return 0; -} - -#endif /* #ifdef _WIN32 */ - -/* - * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories, - * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb). - * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) - * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. - */ -UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb) -{ - size_t pos; - unsigned i, nbFiles; - char* buf = (char*)malloc(LIST_SIZE_INCREASE); - char* bufend = buf + LIST_SIZE_INCREASE; - const char** fileTable; - - if (!buf) return NULL; - - for (i=0, pos=0, nbFiles=0; i= bufend) { - ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; - buf = (char*)UTIL_realloc(buf, newListSize); - bufend = buf + newListSize; - if (!buf) return NULL; - } - if (buf + pos + len < bufend) { - strncpy(buf + pos, inputNames[i], bufend - (buf + pos)); - pos += len + 1; - nbFiles++; - } - } else { - nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend); - if (buf == NULL) return NULL; - } } - - if (nbFiles == 0) { free(buf); return NULL; } - - fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); - if (!fileTable) { free(buf); return NULL; } - - for (i=0, pos=0; i bufend) { free(buf); free((void*)fileTable); return NULL; } - - *allocatedBuffer = buf; - *allocatedNamesNb = nbFiles; - - return fileTable; -} - - -UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer) -{ - if (allocatedBuffer) free(allocatedBuffer); - if (filenameTable) free((void*)filenameTable); -} - - -#if defined (__cplusplus) -} -#endif - -#endif /* UTIL_H_MODULE */ diff --git a/contrib/lizard/tests/.gitignore b/contrib/lizard/tests/.gitignore deleted file mode 100644 index 06891acf794..00000000000 --- a/contrib/lizard/tests/.gitignore +++ /dev/null @@ -1,13 +0,0 @@ - -# test build artefacts -datagen -frametest -frametest32 -fullbench -fullbench32 -fuzzer -fuzzer32 -fasttest - -# test artefacts -tmp* diff --git a/contrib/lizard/tests/COPYING b/contrib/lizard/tests/COPYING deleted file mode 100644 index d159169d105..00000000000 --- a/contrib/lizard/tests/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/contrib/lizard/tests/README.md b/contrib/lizard/tests/README.md deleted file mode 100644 index 4dc2edc4396..00000000000 --- a/contrib/lizard/tests/README.md +++ /dev/null @@ -1,15 +0,0 @@ -Programs and scripts for automated testing of Lizard -======================================================= - -This directory contains the following programs and scripts: -- `datagen` : Synthetic and parametrable data generator, for tests -- `frametest` : Test tool that checks lizard_frame integrity on target platform -- `fullbench` : Precisely measure speed for each lizard inner functions -- `fuzzer` : Test tool, to check lizard integrity on target platform - - -#### License - -All files in this directory are licensed under GPL-v2. -See [COPYING](COPYING) for details. -The text of the license is also included at the top of each source file. diff --git a/contrib/lizard/tests/datagencli.c b/contrib/lizard/tests/datagencli.c deleted file mode 100644 index 47c79d31522..00000000000 --- a/contrib/lizard/tests/datagencli.c +++ /dev/null @@ -1,192 +0,0 @@ -/* - datagencli.c - compressible data command line generator - Copyright (C) Yann Collet 2012-2015 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ - -/************************************** -* Includes -**************************************/ -#include /* fprintf, stderr */ -#include "datagen.h" /* RDG_generate */ - - -/************************************** -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -/************************************** -* Constants -**************************************/ -#ifndef ZSTD_VERSION -# define ZSTD_VERSION "r1" -#endif - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define SIZE_DEFAULT (64 KB) -#define SEED_DEFAULT 0 -#define COMPRESSIBILITY_DEFAULT 50 - - -/************************************** -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned displayLevel = 2; - - -/********************************************************* -* Command line -*********************************************************/ -static int usage(char* programName) -{ - DISPLAY( "Compressible data generator\n"); - DISPLAY( "Usage :\n"); - DISPLAY( " %s [size] [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); - DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); - DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT); - DISPLAY( " -h : display help and exit\n"); - DISPLAY( "Special values :\n"); - DISPLAY( " -P0 : generate incompressible noise\n"); - DISPLAY( " -P100 : generate sparse files\n"); - return 0; -} - - -int main(int argc, char** argv) -{ - int argNb; - double proba = (double)COMPRESSIBILITY_DEFAULT / 100; - double litProba = 0.0; - U64 size = SIZE_DEFAULT; - U32 seed = SEED_DEFAULT; - char* programName; - - /* Check command line */ - programName = argv[0]; - for(argNb=1; argNb='0') && (*argument<='9')) - { - size *= 10; - size += *argument - '0'; - argument++; - } - if (*argument=='K') { size <<= 10; argument++; } - if (*argument=='M') { size <<= 20; argument++; } - if (*argument=='G') { size <<= 30; argument++; } - if (*argument=='B') { argument++; } - break; - case 's': - argument++; - seed=0; - while ((*argument>='0') && (*argument<='9')) - { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - case 'P': - argument++; - proba=0.0; - while ((*argument>='0') && (*argument<='9')) - { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba>100.) proba=100.; - proba /= 100.; - break; - case 'L': /* hidden argument : Literal distribution probability */ - argument++; - litProba=0.; - while ((*argument>='0') && (*argument<='9')) - { - litProba *= 10; - litProba += *argument - '0'; - argument++; - } - if (litProba>100.) litProba=100.; - litProba /= 100.; - break; - case 'v': - displayLevel = 4; - argument++; - break; - default: - return usage(programName); - } - } - - } - } - - DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION); - DISPLAYLEVEL(3, "Seed = %u \n", seed); - if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100)); - - RDG_genOut(size, proba, litProba, seed); - DISPLAYLEVEL(1, "\n"); - - return 0; -} diff --git a/contrib/lizard/tests/frametest.c b/contrib/lizard/tests/frametest.c deleted file mode 100644 index 8fe64869fa4..00000000000 --- a/contrib/lizard/tests/frametest.c +++ /dev/null @@ -1,866 +0,0 @@ -/* - frameTest - test tool for lizard_frame - Copyright (C) Yann Collet 2014-2016 - Copyright (C) Przemyslaw Skibinski 2016-2017 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ - -/*-************************************ -* Compiler specific -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ -#endif - - -/*-************************************ -* Includes -**************************************/ -#include /* malloc, free */ -#include /* fprintf */ -#include /* strcmp */ -#include /* clock_t, clock(), CLOCKS_PER_SEC */ -#include "lizard_frame_static.h" -#include "lizard_compress.h" /* LIZARD_VERSION_STRING */ -#define XXH_STATIC_LINKING_ONLY -#include "xxhash/xxhash.h" /* XXH64 */ - - -/*-************************************ -* Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - - -/* unoptimized version; solves endianess & alignment issues */ -static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) -{ - BYTE* dstPtr = (BYTE*)dstVoidPtr; - dstPtr[0] = (BYTE)value32; - dstPtr[1] = (BYTE)(value32 >> 8); - dstPtr[2] = (BYTE)(value32 >> 16); - dstPtr[3] = (BYTE)(value32 >> 24); -} - - -/*-************************************ -* Constants -**************************************/ -#define LIZARDF_MAGIC_SKIPPABLE_START 0x184D2A50U - -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - -static const U32 nbTestsDefault = 256 KB; -#define COMPRESSIBLE_NOISE_LENGTH (2 MB) -#define FUZ_COMPRESSIBILITY_DEFAULT 50 -static const U32 prime1 = 2654435761U; -static const U32 prime2 = 2246822519U; - - - -/*-************************************ -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \ - if ((FUZ_GetClockSpan(g_clockTime) > refreshRate) || (displayLevel>=4)) \ - { g_clockTime = clock(); DISPLAY(__VA_ARGS__); \ - if (displayLevel>=4) fflush(stdout); } } -static const clock_t refreshRate = CLOCKS_PER_SEC / 6; -static clock_t g_clockTime = 0; - - -/*-*************************************** -* Local Parameters -*****************************************/ -static U32 no_prompt = 0; -static char* programName; -static U32 displayLevel = 2; -static U32 pause = 0; - - -/*-******************************************************* -* Fuzzer functions -*********************************************************/ -static clock_t FUZ_GetClockSpan(clock_t clockStart) -{ - return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */ -} - - -#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) -unsigned int FUZ_rand(unsigned int* src) -{ - U32 rand32 = *src; - rand32 *= prime1; - rand32 += prime2; - rand32 = FUZ_rotl32(rand32, 13); - *src = rand32; - return rand32 >> 5; -} - - -#define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF) -#define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) -static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, double proba, U32* seed) -{ - BYTE* BBuffer = (BYTE*)buffer; - unsigned pos = 0; - U32 P32 = (U32)(32768 * proba); - - /* First Byte */ - BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); - - while (pos < bufferSize) { - /* Select : Literal (noise) or copy (within 64K) */ - if (FUZ_RAND15BITS < P32) { - /* Copy (within 64K) */ - unsigned match, end; - unsigned length = FUZ_RANDLENGTH + 4; - unsigned offset = FUZ_RAND15BITS + 1; - if (offset > pos) offset = pos; - if (pos + length > bufferSize) length = bufferSize - pos; - match = pos - offset; - end = pos + length; - while (pos < end) BBuffer[pos++] = BBuffer[match++]; - } else { - /* Literal (noise) */ - unsigned end; - unsigned length = FUZ_RANDLENGTH; - if (pos + length > bufferSize) length = bufferSize - pos; - end = pos + length; - while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); - } - } -} - - -static unsigned FUZ_highbit(U32 v32) -{ - unsigned nbBits = 0; - if (v32==0) return 0; - while (v32) v32 >>= 1, nbBits ++; - return nbBits; -} - - -int basicTests(U32 seed, double compressibility) -{ - int testResult = 0; - void* CNBuffer; - void* compressedBuffer; - void* decodedBuffer; - U32 randState = seed; - size_t cSize, testSize; - LizardF_preferences_t prefs; - LizardF_decompressionContext_t dCtx = NULL; - LizardF_compressionContext_t cctx = NULL; - U64 crcOrig; - - /* Create compressible test buffer */ - memset(&prefs, 0, sizeof(prefs)); - CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - compressedBuffer = malloc(LizardF_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL)); - decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); - crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - - /* Special case : null-content frame */ - testSize = 0; - DISPLAYLEVEL(3, "LizardF_compressFrame, compress null content : \n"); - cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); - if (LizardF_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize); - - DISPLAYLEVEL(3, "LizardF_createDecompressionContext \n"); - { LizardF_errorCode_t const errorCode = LizardF_createDecompressionContext(&dCtx, LIZARDF_VERSION); - if (LizardF_isError(errorCode)) goto _output_error; } - - DISPLAYLEVEL(3, "LizardF_getFrameInfo on null-content frame (#157) \n"); - { size_t avail_in = cSize; - LizardF_frameInfo_t frame_info; - LizardF_errorCode_t const errorCode = LizardF_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in); - if (LizardF_isError(errorCode)) goto _output_error; - } - - DISPLAYLEVEL(3, "LizardF_freeDecompressionContext \n"); - { LizardF_errorCode_t const errorCode = LizardF_freeDecompressionContext(dCtx); - if (LizardF_isError(errorCode)) goto _output_error; } - - /* Trivial tests : one-step frame */ - testSize = COMPRESSIBLE_NOISE_LENGTH; - DISPLAYLEVEL(3, "LizardF_compressFrame, using default preferences : \n"); - cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); - if (LizardF_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Decompression test : \n"); - { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; - size_t compressedBufferSize = cSize; - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; - BYTE* ip = (BYTE*)compressedBuffer; - BYTE* const iend = (BYTE*)compressedBuffer + cSize; - U64 crcDest; - - LizardF_errorCode_t errorCode = LizardF_createDecompressionContext(&dCtx, LIZARDF_VERSION); - if (LizardF_isError(errorCode)) goto _output_error; - - DISPLAYLEVEL(3, "Single Block : \n"); - errorCode = LizardF_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL); - if (LizardF_isError(errorCode)) goto _output_error; - crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; - DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); - - DISPLAYLEVEL(3, "Reusing decompression context \n"); - { size_t iSize = compressedBufferSize - 4; - const BYTE* cBuff = (const BYTE*) compressedBuffer; - size_t decResult; - DISPLAYLEVEL(3, "Missing last 4 bytes : "); - decResult = LizardF_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL); - if (LizardF_isError(decResult)) goto _output_error; - if (!decResult) goto _output_error; /* not finished */ - DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned)decResult); - cBuff += iSize; - iSize = decResult; - decResult = LizardF_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL); - if (decResult != 0) goto _output_error; /* should finish now */ - crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; - } - - { size_t oSize = 0; - size_t iSize = 0; - LizardF_frameInfo_t fi; - - DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); - errorCode = LizardF_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL); - if (LizardF_isError(errorCode)) goto _output_error; - DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); - - DISPLAYLEVEL(3, "get FrameInfo on null input : "); - errorCode = LizardF_getFrameInfo(dCtx, &fi, ip, &iSize); - if (errorCode != (size_t)-LizardF_ERROR_frameHeader_incomplete) goto _output_error; - DISPLAYLEVEL(3, " correctly failed : %s \n", LizardF_getErrorName(errorCode)); - - DISPLAYLEVEL(3, "get FrameInfo on not enough input : "); - iSize = 6; - errorCode = LizardF_getFrameInfo(dCtx, &fi, ip, &iSize); - if (errorCode != (size_t)-LizardF_ERROR_frameHeader_incomplete) goto _output_error; - DISPLAYLEVEL(3, " correctly failed : %s \n", LizardF_getErrorName(errorCode)); - ip += iSize; - - DISPLAYLEVEL(3, "get FrameInfo on enough input : "); - iSize = 15 - iSize; - errorCode = LizardF_getFrameInfo(dCtx, &fi, ip, &iSize); - if (LizardF_isError(errorCode)) goto _output_error; - DISPLAYLEVEL(3, " correctly decoded \n"); - ip += iSize; - } - - DISPLAYLEVEL(3, "Byte after byte : \n"); - while (ip < iend) { - size_t oSize = oend-op; - size_t iSize = 1; - errorCode = LizardF_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LizardF_isError(errorCode)) goto _output_error; - op += oSize; - ip += iSize; - } - crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; - DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-(BYTE*)decodedBuffer), COMPRESSIBLE_NOISE_LENGTH); - - errorCode = LizardF_freeDecompressionContext(dCtx); - if (LizardF_isError(errorCode)) goto _output_error; - } - - DISPLAYLEVEL(3, "Using 128 KB block : \n"); - prefs.frameInfo.blockSizeID = LizardF_max128KB; - prefs.frameInfo.contentChecksumFlag = LizardF_contentChecksumEnabled; - cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LizardF_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = LizardF_noContentChecksum; - cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LizardF_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Using 256 KB block : \n"); - prefs.frameInfo.blockSizeID = LizardF_max256KB; - prefs.frameInfo.contentChecksumFlag = LizardF_contentChecksumEnabled; - cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LizardF_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Decompression test : \n"); - { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; - unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; - BYTE* ip = (BYTE*)compressedBuffer; - BYTE* const iend = (BYTE*)compressedBuffer + cSize; - U64 crcDest; - - LizardF_errorCode_t errorCode = LizardF_createDecompressionContext(&dCtx, LIZARDF_VERSION); - if (LizardF_isError(errorCode)) goto _output_error; - - DISPLAYLEVEL(3, "random segment sizes : \n"); - while (ip < iend) { - unsigned nbBits = FUZ_rand(&randState) % maxBits; - size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; - //DISPLAY("%7i : + %6i\n", (int)(ip-(BYTE*)compressedBuffer), (int)iSize); - errorCode = LizardF_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LizardF_isError(errorCode)) goto _output_error; - op += oSize; - ip += iSize; - } - crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - if (crcDest != crcOrig) goto _output_error; - DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); - - errorCode = LizardF_freeDecompressionContext(dCtx); - if (LizardF_isError(errorCode)) goto _output_error; - } - - DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = LizardF_noContentChecksum; - cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LizardF_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Using 1 MB block : \n"); - prefs.frameInfo.blockSizeID = LizardF_max1MB; - prefs.frameInfo.contentChecksumFlag = LizardF_contentChecksumEnabled; - cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LizardF_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = LizardF_noContentChecksum; - cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LizardF_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "Using 4 MB block : \n"); - prefs.frameInfo.blockSizeID = LizardF_max4MB; - prefs.frameInfo.contentChecksumFlag = LizardF_contentChecksumEnabled; - cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LizardF_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = LizardF_noContentChecksum; - cSize = LizardF_compressFrame(compressedBuffer, LizardF_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); - if (LizardF_isError(cSize)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); - - { size_t errorCode; - BYTE* const ostart = (BYTE*)compressedBuffer; - BYTE* op = ostart; - errorCode = LizardF_createCompressionContext(&cctx, LIZARDF_VERSION); - if (LizardF_isError(errorCode)) goto _output_error; - - DISPLAYLEVEL(3, "compress without frameSize : \n"); - memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo)); - errorCode = LizardF_compressBegin(cctx, compressedBuffer, testSize, &prefs); - if (LizardF_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LizardF_compressUpdate(cctx, op, LizardF_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); - if (LizardF_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LizardF_compressEnd(cctx, compressedBuffer, testSize, NULL); - if (LizardF_isError(errorCode)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); - - DISPLAYLEVEL(3, "compress with frameSize : \n"); - prefs.frameInfo.contentSize = testSize; - op = ostart; - errorCode = LizardF_compressBegin(cctx, compressedBuffer, testSize, &prefs); - if (LizardF_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LizardF_compressUpdate(cctx, op, LizardF_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); - if (LizardF_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LizardF_compressEnd(cctx, compressedBuffer, testSize, NULL); - if (LizardF_isError(errorCode)) goto _output_error; - DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); - - DISPLAYLEVEL(3, "compress with wrong frameSize : \n"); - prefs.frameInfo.contentSize = testSize+1; - op = ostart; - errorCode = LizardF_compressBegin(cctx, compressedBuffer, testSize, &prefs); - if (LizardF_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LizardF_compressUpdate(cctx, op, LizardF_compressBound(testSize, &prefs), CNBuffer, testSize, NULL); - if (LizardF_isError(errorCode)) goto _output_error; - op += errorCode; - errorCode = LizardF_compressEnd(cctx, op, testSize, NULL); - if (LizardF_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LizardF_getErrorName(errorCode)); } - else - goto _output_error; - - errorCode = LizardF_freeCompressionContext(cctx); - if (LizardF_isError(errorCode)) goto _output_error; - cctx = NULL; - } - - DISPLAYLEVEL(3, "Skippable frame test : \n"); - { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; - unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; - BYTE* ip = (BYTE*)compressedBuffer; - BYTE* iend = (BYTE*)compressedBuffer + cSize + 8; - - LizardF_errorCode_t errorCode = LizardF_createDecompressionContext(&dCtx, LIZARDF_VERSION); - if (LizardF_isError(errorCode)) goto _output_error; - - /* generate skippable frame */ - FUZ_writeLE32(ip, LIZARDF_MAGIC_SKIPPABLE_START); - FUZ_writeLE32(ip+4, (U32)cSize); - - DISPLAYLEVEL(3, "random segment sizes : \n"); - while (ip < iend) { - unsigned nbBits = FUZ_rand(&randState) % maxBits; - size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; - errorCode = LizardF_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LizardF_isError(errorCode)) goto _output_error; - op += oSize; - ip += iSize; - } - DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize); - - /* generate zero-size skippable frame */ - DISPLAYLEVEL(3, "zero-size skippable frame\n"); - ip = (BYTE*)compressedBuffer; - op = (BYTE*)decodedBuffer; - FUZ_writeLE32(ip, LIZARDF_MAGIC_SKIPPABLE_START+1); - FUZ_writeLE32(ip+4, 0); - iend = ip+8; - - while (ip < iend) { - unsigned nbBits = FUZ_rand(&randState) % maxBits; - size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; - errorCode = LizardF_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LizardF_isError(errorCode)) goto _output_error; - op += oSize; - ip += iSize; - } - DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); - - DISPLAYLEVEL(3, "Skippable frame header complete in first call \n"); - ip = (BYTE*)compressedBuffer; - op = (BYTE*)decodedBuffer; - FUZ_writeLE32(ip, LIZARDF_MAGIC_SKIPPABLE_START+2); - FUZ_writeLE32(ip+4, 10); - iend = ip+18; - while (ip < iend) { - size_t iSize = 10; - size_t oSize = 10; - if (iSize > (size_t)(iend-ip)) iSize = iend-ip; - errorCode = LizardF_decompress(dCtx, op, &oSize, ip, &iSize, NULL); - if (LizardF_isError(errorCode)) goto _output_error; - op += oSize; - ip += iSize; - } - DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); - } - - DISPLAY("Basic tests completed \n"); -_end: - free(CNBuffer); - free(compressedBuffer); - free(decodedBuffer); - LizardF_freeDecompressionContext(dCtx); dCtx = NULL; - LizardF_freeCompressionContext(cctx); cctx = NULL; - return testResult; - -_output_error: - testResult = 1; - DISPLAY("Error detected ! \n"); - goto _end; -} - - -static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous) -{ - int p=0; - const BYTE* b1=(const BYTE*)buff1; - const BYTE* b2=(const BYTE*)buff2; - if (nonContiguous) { - DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size); - return; - } - while (b1[p]==b2[p]) p++; - DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]); -} - - -static const U32 srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ - -int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s) -{ - unsigned testResult = 0; - unsigned testNb = 0; - void* srcBuffer = NULL; - void* compressedBuffer = NULL; - void* decodedBuffer = NULL; - U32 coreRand = seed; - LizardF_decompressionContext_t dCtx = NULL; - LizardF_compressionContext_t cCtx = NULL; - size_t result; - clock_t const startClock = clock(); - clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; - XXH64_state_t xxh64; -# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } - - /* Create buffers */ - result = LizardF_createDecompressionContext(&dCtx, LIZARDF_VERSION); - CHECK(LizardF_isError(result), "Allocation failed (error %i)", (int)result); - result = LizardF_createCompressionContext(&cCtx, LIZARDF_VERSION); - CHECK(LizardF_isError(result), "Allocation failed (error %i)", (int)result); - srcBuffer = malloc(srcDataLength); - CHECK(srcBuffer==NULL, "srcBuffer Allocation failed"); - compressedBuffer = malloc(LizardF_compressFrameBound(srcDataLength, NULL)); - CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); - decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ - CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); - FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); - - /* jump to requested testNb */ - for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer - - /* main fuzzer test loop */ - for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) { - U32 randState = coreRand ^ prime1; - unsigned BSId = 4 + (FUZ_rand(&randState) & 3); - unsigned BMId = FUZ_rand(&randState) & 1; - unsigned CCflag = FUZ_rand(&randState) & 1; - unsigned autoflush = (FUZ_rand(&randState) & 7) == 2; - LizardF_preferences_t prefs; - LizardF_compressOptions_t cOptions; - LizardF_decompressOptions_t dOptions; - unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1; - size_t srcSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; - cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); - - result = LizardF_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); - CHECK(LizardF_isError(result), "Compression failed (error %i) iSize=%d oSize=%d", (int)result, (int)iSize, (int)oSize); - op += result; - ip += iSize; - - if (forceFlush) { - result = LizardF_flush(cCtx, op, oend-op, &cOptions); - CHECK(LizardF_isError(result), "Compression flush failed (error %i)", (int)result); - op += result; - } - } - result = LizardF_compressEnd(cCtx, op, oend-op, &cOptions); - CHECK(LizardF_isError(result), "Compression completion failed (error %i)", (int)result); - op += result; - cSize = op-(BYTE*)compressedBuffer; - } - - { const BYTE* ip = (const BYTE*)compressedBuffer; - const BYTE* const iend = ip + cSize; - BYTE* op = (BYTE*)decodedBuffer; - BYTE* const oend = op + srcDataLength; - size_t totalOut = 0; - unsigned maxBits = FUZ_highbit((U32)cSize); - unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1; - nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst; /* 0=>0; 1=>1,2 */ - XXH64_reset(&xxh64, 1); - if (maxBits < 3) maxBits = 3; - while (ip < iend) { - unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; - unsigned nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1; - size_t iSize = (FUZ_rand(&randState) & ((1< (size_t)(iend-ip)) iSize = iend-ip; - if (oSize > (size_t)(oend-op)) oSize = oend-op; - dOptions.stableDst = FUZ_rand(&randState) & 1; - if (nonContiguousDst==2) dOptions.stableDst = 0; - result = LizardF_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); - if (result == (size_t)-LizardF_ERROR_contentChecksum_invalid) - locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); - CHECK(LizardF_isError(result), "Decompression failed (error %i:%s)", (int)result, LizardF_getErrorName((LizardF_errorCode_t)result)); - XXH64_update(&xxh64, op, (U32)oSize); - totalOut += oSize; - op += oSize; - ip += iSize; - op += nonContiguousDst; - if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */ - } - CHECK(result != 0, "Frame decompression failed (error %i)", (int)result); - if (totalOut) { /* otherwise, it's a skippable frame */ - crcDecoded = XXH64_digest(&xxh64); - if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); - CHECK(crcDecoded != crcOrig, "Decompression corruption"); - } - } - } - - DISPLAYLEVEL(2, "\rAll tests completed \n"); - -_end: - LizardF_freeDecompressionContext(dCtx); - LizardF_freeCompressionContext(cCtx); - free(srcBuffer); - free(compressedBuffer); - free(decodedBuffer); - - if (pause) { - DISPLAY("press enter to finish \n"); - (void)getchar(); - } - return testResult; - -_output_error: - testResult = 1; - goto _end; -} - - -int FUZ_usage(void) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); - DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); - DISPLAY( " -s# : Select seed (default:prompt user)\n"); - DISPLAY( " -t# : Select starting test number (default:0)\n"); - DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); - DISPLAY( " -v : verbose\n"); - DISPLAY( " -h : display help and exit\n"); - return 0; -} - - -int main(int argc, char** argv) -{ - U32 seed=0; - int seedset=0; - int argNb; - int nbTests = nbTestsDefault; - int testNb = 0; - int proba = FUZ_COMPRESSIBILITY_DEFAULT; - int result=0; - U32 duration=0; - - /* Check command line */ - programName = argv[0]; - for(argNb=1; argNb='0') && (*argument<='9')) { - nbTests *= 10; - nbTests += *argument - '0'; - argument++; - } - break; - - case 'T': - argument++; - nbTests = 0; duration = 0; - for (;;) { - switch(*argument) - { - case 'm': duration *= 60; argument++; continue; - case 's': - case 'n': argument++; continue; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': duration *= 10; duration += *argument++ - '0'; continue; - } - break; - } - break; - - case 's': - argument++; - seed=0; - seedset=1; - while ((*argument>='0') && (*argument<='9')) { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - case 't': - argument++; - testNb=0; - while ((*argument>='0') && (*argument<='9')) { - testNb *= 10; - testNb += *argument - '0'; - argument++; - } - break; - case 'P': /* compressibility % */ - argument++; - proba=0; - while ((*argument>='0') && (*argument<='9')) { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba<0) proba=0; - if (proba>100) proba=100; - break; - default: - ; - return FUZ_usage(); - } - } - } - } - - /* Get Seed */ - printf("Starting lizard_frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LIZARD_VERSION_STRING); - - if (!seedset) { - time_t const t = time(NULL); - U32 const h = XXH32(&t, sizeof(t), 1); - seed = h % 10000; - } - printf("Seed = %u\n", seed); - if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); - - if (nbTests<=0) nbTests=1; - - if (testNb==0) result = basicTests(seed, ((double)proba) / 100); - if (result) return 1; - return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration); -} diff --git a/contrib/lizard/tests/fullbench.c b/contrib/lizard/tests/fullbench.c deleted file mode 100644 index d9070d13345..00000000000 --- a/contrib/lizard/tests/fullbench.c +++ /dev/null @@ -1,732 +0,0 @@ -/* - bench.c - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2015 - Copyright (C) Przemyslaw Skibinski 2016-2017 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - Lizard source repository : https://github.com/inikep/lizard -*/ - -/************************************** -* Compiler Options -**************************************/ -/* Disable some Visual warning messages */ -#define _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ - -/* Unix Large Files support (>4GB) */ -#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions -# define _LARGEFILE_SOURCE -# define _FILE_OFFSET_BITS 64 -#elif ! defined(__LP64__) // No point defining Large file for 64 bit -# define _LARGEFILE64_SOURCE -#endif - -// S_ISREG & gettimeofday() are not supported by MSVC -#if defined(_MSC_VER) || defined(_WIN32) -# define BMK_LEGACY_TIMER 1 -#endif - - -/************************************** -* Includes -**************************************/ -#include /* malloc, free */ -#include /* fprintf, fopen, ftello64 */ -#include /* stat64 */ -#include /* stat64 */ -#include /* strcmp */ -#include /* clock_t, clock(), CLOCKS_PER_SEC */ - -#include "lizard_compress.h" -#include "lizard_decompress.h" -#include "lizard_common.h" /* Lizard_compress_MinLevel, Lizard_createStream_MinLevel */ -#include "lizard_frame.h" - -#include "xxhash/xxhash.h" - - -/************************************** -* Compiler Options -**************************************/ -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - - - -/************************************** -* Constants -**************************************/ -#define PROGRAM_DESCRIPTION "Lizard speed analyzer" -#define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s v%s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, LIZARD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR - -#define NBLOOPS 6 -#define TIMELOOP (CLOCKS_PER_SEC * 25 / 10) - -#define KNUTH 2654435761U -#define MAX_MEM (1920 MB) -#define DEFAULT_CHUNKSIZE (4 MB) - -#define ALL_COMPRESSORS 0 -#define ALL_DECOMPRESSORS 0 - - -/************************************** -* Local structures -**************************************/ -struct chunkParameters -{ - U32 id; - char* origBuffer; - char* compressedBuffer; - int origSize; - int compressedSize; -}; - - -/************************************** -* Macros -**************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define PROGRESS(...) g_noPrompt ? 0 : DISPLAY(__VA_ARGS__) - - -/************************************** -* Benchmark Parameters -**************************************/ -static int g_chunkSize = DEFAULT_CHUNKSIZE; -static int g_nbIterations = NBLOOPS; -static int g_pause = 0; -static int g_compressionTest = 1; -static int g_compressionAlgo = ALL_COMPRESSORS; -static int g_decompressionTest = 1; -static int g_decompressionAlgo = ALL_DECOMPRESSORS; -static int g_noPrompt = 0; - -static void BMK_setBlocksize(int bsize) -{ - g_chunkSize = bsize; - DISPLAY("-Using Block Size of %i KB-\n", g_chunkSize>>10); -} - -static void BMK_setNbIterations(int nbLoops) -{ - g_nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", g_nbIterations); -} - -static void BMK_setPause(void) -{ - g_pause = 1; -} - - -/********************************************************* -* Private functions -*********************************************************/ -static clock_t BMK_GetClockSpan( clock_t clockStart ) -{ - return clock() - clockStart; /* works even if overflow; max span ~30 mn */ -} - - -static size_t BMK_findMaxMem(U64 requiredMem) -{ - size_t step = 64 MB; - BYTE* testmem=NULL; - - requiredMem = (((requiredMem >> 26) + 1) << 26); - requiredMem += 2*step; - if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; - - while (!testmem) { - if (requiredMem > step) requiredMem -= step; - else requiredMem >>= 1; - testmem = (BYTE*) malloc ((size_t)requiredMem); - } - free (testmem); - - /* keep some space available */ - if (requiredMem > step) requiredMem -= step; - else requiredMem >>= 1; - - return (size_t)requiredMem; -} - - -static U64 BMK_GetFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - - -/********************************************************* -* Benchmark function -*********************************************************/ -Lizard_stream_t *Lizard_stream; -static void local_Lizard_createStream(void) -{ - Lizard_stream = Lizard_resetStream_MinLevel(Lizard_stream); -} - -static int local_Lizard_saveDict(const char* in, char* out, int inSize) -{ - (void)in; - return Lizard_saveDict(Lizard_stream, out, inSize); -} - -static int local_Lizard_compress_default_large(const char* in, char* out, int inSize) -{ - return Lizard_compress_MinLevel(in, out, inSize, Lizard_compressBound(inSize)); -} - -static int local_Lizard_compress_default_small(const char* in, char* out, int inSize) -{ - return Lizard_compress_MinLevel(in, out, inSize, Lizard_compressBound(inSize)-1); -} - -static int local_Lizard_compress_withState(const char* in, char* out, int inSize) -{ - return Lizard_compress_extState_MinLevel(Lizard_stream, in, out, inSize, Lizard_compressBound(inSize)); -} - -static int local_Lizard_compress_limitedOutput_withState(const char* in, char* out, int inSize) -{ - return Lizard_compress_extState_MinLevel(Lizard_stream, in, out, inSize, Lizard_compressBound(inSize)-1); -} - -static int local_Lizard_compress_continue(const char* in, char* out, int inSize) -{ - return Lizard_compress_continue(Lizard_stream, in, out, inSize, Lizard_compressBound(inSize)); -} - -static int local_Lizard_compress_limitedOutput_continue(const char* in, char* out, int inSize) -{ - return Lizard_compress_continue(Lizard_stream, in, out, inSize, Lizard_compressBound(inSize)-1); -} - - -/* HC compression functions */ -Lizard_stream_t* Lizard_streamPtr; -static void local_Lizard_resetStream(void) -{ - Lizard_streamPtr = Lizard_resetStream(Lizard_streamPtr, 0); -} - -static int local_Lizard_saveDictHC(const char* in, char* out, int inSize) -{ - (void)in; - return Lizard_saveDict(Lizard_streamPtr, out, inSize); -} - -static int local_Lizard_compress_extState(const char* in, char* out, int inSize) -{ - return Lizard_compress_extState(Lizard_streamPtr, in, out, inSize, Lizard_compressBound(inSize), 0); -} - -static int local_Lizard_compress_extState_limitedOutput(const char* in, char* out, int inSize) -{ - return Lizard_compress_extState(Lizard_streamPtr, in, out, inSize, Lizard_compressBound(inSize)-1, 0); -} - -static int local_Lizard_compress(const char* in, char* out, int inSize) -{ - return Lizard_compress(in, out, inSize, Lizard_compressBound(inSize), 0); -} - -static int local_Lizard_compress_limitedOutput(const char* in, char* out, int inSize) -{ - return Lizard_compress(in, out, inSize, Lizard_compressBound(inSize)-1, 0); -} - -static int local_Lizard_compressHC_continue(const char* in, char* out, int inSize) -{ - return Lizard_compress_continue(Lizard_streamPtr, in, out, inSize, Lizard_compressBound(inSize)); -} - -static int local_Lizard_compress_continue_limitedOutput(const char* in, char* out, int inSize) -{ - return Lizard_compress_continue(Lizard_streamPtr, in, out, inSize, Lizard_compressBound(inSize)-1); -} - - -/* decompression functions */ -static int local_Lizard_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize) -{ - (void)inSize; - Lizard_decompress_safe_usingDict(in, out, inSize, outSize, out - 65536, 65536); - return outSize; -} - -extern int Lizard_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const char* dict, int dictSize); - -static int local_Lizard_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize) -{ - (void)inSize; - Lizard_decompress_safe_forceExtDict(in, out, inSize, outSize, out - 65536, 65536); - return outSize; -} - -static int local_Lizard_decompress_safe_partial(const char* in, char* out, int inSize, int outSize) -{ - return Lizard_decompress_safe_partial(in, out, inSize, outSize - 5, outSize); -} - - -/* frame functions */ -static int local_LizardF_compressFrame(const char* in, char* out, int inSize) -{ - return (int)LizardF_compressFrame(out, 2*inSize + 16, in, inSize, NULL); -} - -static LizardF_decompressionContext_t g_dCtx; - -static int local_LizardF_decompress(const char* in, char* out, int inSize, int outSize) -{ - size_t srcSize = inSize; - size_t dstSize = outSize; - size_t result; - result = LizardF_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL); - if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame (%d)\n", (int)result); exit(8); } - if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect\n"); exit(9); } - return (int)dstSize; -} - - -#define NB_COMPRESSION_ALGORITHMS 100 -#define NB_DECOMPRESSION_ALGORITHMS 100 -int fullSpeedBench(const char** fileNamesTable, int nbFiles) -{ - int fileIdx=0; - - /* Init */ - { size_t const errorCode = LizardF_createDecompressionContext(&g_dCtx, LIZARDF_VERSION); - if (LizardF_isError(errorCode)) { DISPLAY("dctx allocation issue \n"); return 10; } } - - Lizard_streamPtr = Lizard_createStream(0); - if (!Lizard_streamPtr) { DISPLAY("Lizard_streamPtr allocation issue \n"); return 10; } - - Lizard_stream = Lizard_createStream_MinLevel(); - if (!Lizard_stream) { DISPLAY("Lizard_stream allocation issue \n"); return 10; } - - /* Loop for each fileName */ - while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - - /* Allocation */ - chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)g_chunkSize)+1) * sizeof(struct chunkParameters)); - orig_buff = (char*) malloc(benchedSize); - nbChunks = (int) ((benchedSize + (g_chunkSize-1)) / g_chunkSize); - maxCompressedChunkSize = Lizard_compressBound(g_chunkSize); - compressedBuffSize = nbChunks * maxCompressedChunkSize; - compressed_buff = (char*)malloc((size_t)compressedBuffSize); - if(!chunkP || !orig_buff || !compressed_buff) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - free(orig_buff); - free(compressed_buff); - free(chunkP); - return(12); - } - - /* Fill in src buffer */ - DISPLAY("Loading %s... \r", inFileName); - readSize = fread(orig_buff, 1, benchedSize, inFile); - fclose(inFile); - - if (readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(orig_buff); - free(compressed_buff); - free(chunkP); - return 13; - } - - /* Calculating input Checksum */ - crcOriginal = XXH32(orig_buff, benchedSize,0); - - - /* Bench */ - { int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb; - size_t cSize=0; - double ratio=0.; - - DISPLAY("\r%79s\r", ""); - DISPLAY(" %s : \n", inFileName); - - /* Bench Compression Algorithms */ - for (cAlgNb=0; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (g_compressionTest); cAlgNb++) { - const char* compressorName; - int (*compressionFunction)(const char*, char*, int); - void (*initFunction)(void) = NULL; - double bestTime = 100000000.; - - /* filter compressionAlgo only */ - if ((g_compressionAlgo != ALL_COMPRESSORS) && (g_compressionAlgo != cAlgNb)) continue; - - /* Init data chunks */ - { int i; - size_t remaining = benchedSize; - char* in = orig_buff; - char* out = compressed_buff; - nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); - for (i=0; i g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } - chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; - chunkP[i].compressedSize = 0; - } - } - - switch(cAlgNb) - { - case 0 : DISPLAY("Compression functions : \n"); continue; - case 1 : compressionFunction = local_Lizard_compress_default_large; compressorName = "Lizard_compress_MinLevel"; break; - case 2 : compressionFunction = local_Lizard_compress_default_small; compressorName = "Lizard_compress_MinLevel(small dst)"; break; - - case 10: compressionFunction = local_Lizard_compress; compressorName = "Lizard_compress"; break; - case 11: compressionFunction = local_Lizard_compress_limitedOutput; compressorName = "Lizard_compress limitedOutput"; break; - case 12 : compressionFunction = local_Lizard_compress_extState; compressorName = "Lizard_compress_extState"; break; - case 13: compressionFunction = local_Lizard_compress_extState_limitedOutput; compressorName = "Lizard_compress_extState limitedOutput"; break; - case 14: compressionFunction = local_Lizard_compressHC_continue; initFunction = local_Lizard_resetStream; compressorName = "Lizard_compress_continue"; break; - case 15: compressionFunction = local_Lizard_compress_continue_limitedOutput; initFunction = local_Lizard_resetStream; compressorName = "Lizard_compress_continue limitedOutput"; break; - case 30: compressionFunction = local_LizardF_compressFrame; compressorName = "LizardF_compressFrame"; - chunkP[0].origSize = (int)benchedSize; nbChunks=1; - break; - case 40: compressionFunction = local_Lizard_saveDict; compressorName = "Lizard_saveDict"; - Lizard_loadDict(Lizard_stream, chunkP[0].origBuffer, chunkP[0].origSize); - break; - case 41: compressionFunction = local_Lizard_saveDictHC; compressorName = "Lizard_saveDict"; - Lizard_loadDict(Lizard_streamPtr, chunkP[0].origBuffer, chunkP[0].origSize); - break; - case 16: compressionFunction = local_Lizard_compress_withState; compressorName = "Lizard_compress_extState_MinLevel(1)"; break; - case 17: compressionFunction = local_Lizard_compress_limitedOutput_withState; compressorName = "Lizard_compress_extState_MinLevel(1) limitedOutput"; break; - case 18: compressionFunction = local_Lizard_compress_continue; initFunction = local_Lizard_createStream; compressorName = "Lizard_compress_continue(1)"; break; - case 19: compressionFunction = local_Lizard_compress_limitedOutput_continue; initFunction = local_Lizard_createStream; compressorName = "Lizard_compress_continue(1) limitedOutput"; break; - case 60: DISPLAY("Obsolete compression functions : \n"); continue; - default : - continue; /* unknown ID : just skip */ - } - - for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { - double averageTime; - clock_t clockTime; - - PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize); - { size_t i; for (i=0; i%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000); - } - - if (ratio<100.) - DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000); - else - DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 100000); - } - - /* Prepare layout for decompression */ - /* Init data chunks */ - { int i; - size_t remaining = benchedSize; - char* in = orig_buff; - char* out = compressed_buff; - - nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); - for (i=0; i g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } - chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; - chunkP[i].compressedSize = 0; - } - } - for (chunkNb=0; chunkNb\r", loopNb, dName, (int)benchedSize); - - nb_loops = 0; - clockTime = clock(); - while(clock() == clockTime); - clockTime = clock(); - while(BMK_GetClockSpan(clockTime) < TIMELOOP) { - for (chunkNb=0; chunkNb %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000); - - /* CRC Checking */ - crcDecoded = XXH32(orig_buff, (int)benchedSize, 0); - if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); } - } - - DISPLAY("%2i-%-29.29s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000); - } - } - free(orig_buff); - free(compressed_buff); - free(chunkP); - } - - Lizard_freeStream(Lizard_stream); - Lizard_freeStream(Lizard_streamPtr); - LizardF_freeDecompressionContext(g_dCtx); - if (g_pause) { printf("press enter...\n"); (void)getchar(); } - - return 0; -} - - -static int usage(const char* exename) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); - DISPLAY( "Arguments :\n"); - DISPLAY( " -c : compression tests only\n"); - DISPLAY( " -d : decompression tests only\n"); - DISPLAY( " -H/-h : Help (this text + advanced options)\n"); - return 0; -} - -static int usage_advanced(void) -{ - DISPLAY( "\nAdvanced options :\n"); - DISPLAY( " -c# : test only compression function # [1-%i]\n", NB_COMPRESSION_ALGORITHMS); - DISPLAY( " -d# : test only decompression function # [1-%i]\n", NB_DECOMPRESSION_ALGORITHMS); - DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); - DISPLAY( " -B# : Block size [4-7](default : 7)\n"); - return 0; -} - -static int badusage(const char* exename) -{ - DISPLAY("Wrong parameters\n"); - usage(exename); - return 0; -} - -int main(int argc, const char** argv) -{ - int i, - filenamesStart=2; - const char* exename = argv[0]; - const char* input_filename=0; - - // Welcome message - DISPLAY(WELCOME_MESSAGE); - - if (argc<2) { badusage(exename); return 1; } - - for(i=1; i= '0') && (argument[1]<= '9')) { - g_compressionAlgo *= 10; - g_compressionAlgo += argument[1] - '0'; - argument++; - } - break; - - // Select decompression algorithm only - case 'd': - g_compressionTest = 0; - while ((argument[1]>= '0') && (argument[1]<= '9')) { - g_decompressionAlgo *= 10; - g_decompressionAlgo += argument[1] - '0'; - argument++; - } - break; - - // Display help on usage - case 'h' : - case 'H': usage(exename); usage_advanced(); return 0; - - // Modify Block Properties - case 'B': - while (argument[1]!=0) - switch(argument[1]) - { - case '4': - case '5': - case '6': - case '7': - { int B = argument[1] - '0'; - int S = 1 << (8 + 2*B); - BMK_setBlocksize(S); - argument++; - break; - } - case 'D': argument++; break; - default : goto _exit_blockProperties; - } -_exit_blockProperties: - break; - - // Modify Nb Iterations - case 'i': - if ((argument[1] >='0') && (argument[1] <='9')) { - int iters = argument[1] - '0'; - BMK_setNbIterations(iters); - argument++; - } - break; - - // Pause at the end (hidden option) - case 'p': BMK_setPause(); break; - - // Unknown command - default : badusage(exename); return 1; - } - } - continue; - } - - // first provided filename is input - if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } - - } - - // No input filename ==> Error - if(!input_filename) { badusage(exename); return 1; } - - return fullSpeedBench(argv+filenamesStart, argc-filenamesStart); - -} diff --git a/contrib/lizard/tests/fuzzer.c b/contrib/lizard/tests/fuzzer.c deleted file mode 100644 index 90864dc6d02..00000000000 --- a/contrib/lizard/tests/fuzzer.c +++ /dev/null @@ -1,1086 +0,0 @@ -/* - fuzzer.c - Fuzzer test tool for Lizard - Copyright (C) Yann Collet 2012-2016 - Copyright (C) Przemyslaw Skibinski 2016-2017 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - Lizard source repo : https://github.com/inikep/lizard -*/ - -/*-************************************ -* Compiler options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ -# pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */ -#endif - -/* S_ISREG & gettimeofday() are not supported by MSVC */ -#if defined(_MSC_VER) || defined(_WIN32) -# define FUZ_LEGACY_TIMER 1 -#endif - - -/*-************************************ -* Includes -**************************************/ -#include -#include /* fgets, sscanf */ -#include /* strcmp */ -#include /* clock_t, clock, CLOCKS_PER_SEC */ -#include "lizard_compress.h" /* LIZARD_VERSION_STRING */ -#include "lizard_decompress.h" -#include "lizard_common.h" -#define XXH_STATIC_LINKING_ONLY -#include "xxhash/xxhash.h" - - - -/*-************************************ -* Constants -**************************************/ -#define NB_ATTEMPTS (1<<16) -#define COMPRESSIBLE_NOISE_LENGTH (1 << 21) -#define FUZ_MAX_BLOCK_SIZE (1 << 17) -#define FUZ_MAX_DICT_SIZE (1 << 15) -#define FUZ_COMPRESSIBILITY_DEFAULT 60 -#define PRIME1 2654435761U -#define PRIME2 2246822519U -#define PRIME3 3266489917U - - -/*-*************************************** -* Macros -*****************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static int g_displayLevel = 2; -static const clock_t g_refreshRate = CLOCKS_PER_SEC * 25 / 100; -static clock_t g_time = 0; - - -/*-******************************************************* -* Fuzzer functions -*********************************************************/ -static clock_t FUZ_GetClockSpan(clock_t clockStart) -{ - return clock() - clockStart; /* works even if overflow; max span ~ 30mn */ -} - -static U32 FUZ_rotl32(U32 u32, U32 nbBits) -{ - return ((u32 << nbBits) | (u32 >> (32 - nbBits))); -} - -static U32 FUZ_rand(U32* src) -{ - U32 rand32 = *src; - rand32 *= PRIME1; - rand32 ^= PRIME2; - rand32 = FUZ_rotl32(rand32, 13); - *src = rand32; - return rand32; -} - - -#define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767) -#define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) -static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed) -{ - BYTE* BBuffer = (BYTE*)buffer; - size_t pos = 0; - U32 P32 = (U32)(32768 * proba); - - /* First Bytes */ - while (pos < 20) - BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); - - while (pos < bufferSize) - { - /* Select : Literal (noise) or copy (within 64K) */ - if (FUZ_RAND15BITS < P32) - { - /* Copy (within 64K) */ - size_t match, d; - size_t length = FUZ_RANDLENGTH + 4; - size_t offset = FUZ_RAND15BITS + 1; - while (offset > pos) offset >>= 1; - d = pos + length; - while (d > bufferSize) d = bufferSize; - match = pos - offset; - while (pos < d) BBuffer[pos++] = BBuffer[match++]; - } - else - { - /* Literal (noise) */ - size_t d; - size_t length = FUZ_RANDLENGTH; - d = pos + length; - if (d > bufferSize) d = bufferSize; - while (pos < d) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); - } - } -} - - -#define MAX_NB_BUFF_I134 150 -#define BLOCKSIZE_I134 (32 MB) -/*! FUZ_AddressOverflow() : -* Aggressively pushes memory allocation limits, -* and generates patterns which create address space overflow. -* only possible in 32-bits mode */ -static int FUZ_AddressOverflow(U32* seed) -{ - char* buffers[MAX_NB_BUFF_I134]; - int i, nbBuff=0; - int highAddress = 0; - - DISPLAY("Overflow tests : "); - - /* Only possible in 32-bits */ - /* if (sizeof(void*)==8) - { - DISPLAY("64 bits mode : no overflow \n"); - fflush(stdout); - return 0; - }*/ - - buffers[0] = (char*)malloc(BLOCKSIZE_I134); - buffers[1] = (char*)malloc(BLOCKSIZE_I134); - if ((!buffers[0]) || (!buffers[1])) { - DISPLAY("not enough memory for tests \n"); - return 0; - } - - for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++) { - DISPLAY("%3i \b\b\b\b", nbBuff); - buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134); - if (buffers[nbBuff]==NULL) goto _endOfTests; - - if (((size_t)buffers[nbBuff] > (size_t)0x80000000) && (!highAddress)) { - DISPLAY("high address detected : "); - fflush(stdout); - highAddress=1; - } - - { int const nbOf255 = 1 + (FUZ_rand(seed) % (BLOCKSIZE_I134-1)); - char* const input = buffers[nbBuff-1]; - char* output = buffers[nbBuff]; - int r; - BYTE cLevel = LIZARD_MIN_CLEVEL + (FUZ_rand(seed) % (1+LIZARD_MAX_CLEVEL-LIZARD_MIN_CLEVEL)); - for(i = 5; i < nbOf255; i++) input[i] = (char)0xff; - for(i = 5; i < nbOf255; i+=(FUZ_rand(seed) % 128)) input[i] = (BYTE)(FUZ_rand(seed)%256); - - input[0] = (char)cLevel; /* Compression Level */ - input[1] = (char)0xF0; /* Literal length overflow */ - input[2] = (char)0xFF; - input[3] = (char)0xFF; - input[4] = (char)0xFF; - r = Lizard_decompress_safe(input, output, nbOf255, BLOCKSIZE_I134); - if (r>0 && r0 && r0 && r0 && r g_refreshRate) | (g_displayLevel>=3)) { - g_time = clock(); - DISPLAY("\r%5u ", testNb); - if (g_displayLevel>=3) fflush(stdout); - } -} - - -/*! FUZ_findDiff() : -* find the first different byte between buff1 and buff2. -* presumes buff1 != buff2. -* presumes a difference exists before end of either buffer. -* Typically invoked after a checksum mismatch. -*/ -static void FUZ_findDiff(const void* buff1, const void* buff2) -{ - const BYTE* const b1 = (const BYTE*)buff1; - const BYTE* const b2 = (const BYTE*)buff2; - size_t i=0; - while (b1[i]==b2[i]) i++; - DISPLAY("Wrong Byte at position %u\n", (unsigned)i); -} - - -static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration_s) -{ - unsigned long long bytes = 0; - unsigned long long cbytes = 0; - unsigned long long hcbytes = 0; - unsigned long long ccbytes = 0; - void* CNBuffer; - char* compressedBuffer; - char* decodedBuffer; -# define FUZ_max LIZARD_COMPRESSBOUND(LEN) - int ret; - unsigned cycleNb; -# define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ - printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } -# define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); } - void* stateLizard = malloc(Lizard_sizeofState_MinLevel()); - void* stateLizardHC = malloc(Lizard_sizeofState(0)); - Lizard_stream_t* Lizarddict; - Lizard_stream_t* Lizard_streamHCPtr; - U32 crcOrig, crcCheck; - U32 coreRandState = seed; - U32 randState = coreRandState ^ PRIME3; - int result = 0; - clock_t const clockStart = clock(); - clock_t const clockDuration = (clock_t)duration_s * CLOCKS_PER_SEC; - - - /* init */ - Lizard_streamHCPtr = Lizard_createStream(0); - Lizarddict = Lizard_createStream_MinLevel(); - - /* Create compressible test buffer */ - CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - compressedBuffer = (char*)malloc(Lizard_compressBound(FUZ_MAX_BLOCK_SIZE)); - decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); - - if (!stateLizard || !stateLizardHC || !Lizarddict || !Lizard_streamHCPtr || !CNBuffer || !compressedBuffer || !decodedBuffer) goto _output_error; - - FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); - - /* move to startCycle */ - for (cycleNb = 0; cycleNb < startCycle; cycleNb++) { - (void)FUZ_rand(&coreRandState); - - if (0) { /* some problems can be related to dictionary re-use; in this case, enable this loop */ - int dictSize, blockSize, blockStart; - char* dict; - char* block; - FUZ_displayUpdate(cycleNb); - randState = coreRandState ^ PRIME3; - blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; - blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); - dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; - if (dictSize > blockStart) dictSize = blockStart; - block = ((char*)CNBuffer) + blockStart; - dict = block - dictSize; - Lizard_loadDict(Lizarddict, dict, dictSize); - Lizard_compress_continue(Lizarddict, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); - Lizard_loadDict(Lizarddict, dict, dictSize); - Lizard_compress_continue(Lizarddict, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); - Lizard_loadDict(Lizarddict, dict, dictSize); - Lizard_compress_continue(Lizarddict, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); - } } - - /* Main test loop */ - for (cycleNb = startCycle; (cycleNb < nbCycles) || (FUZ_GetClockSpan(clockStart) < clockDuration) ; cycleNb++) { - U32 testNb = 0; - char* dict; - char* block; - int dictSize, blockSize, blockStart, compressedSize, HCcompressedSize; - int blockContinueCompressedSize; - - FUZ_displayUpdate(cycleNb); - (void)FUZ_rand(&coreRandState); - randState = coreRandState ^ PRIME3; - - /* Select block to test */ - blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1; - blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); - dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; - if (dictSize > blockStart) dictSize = blockStart; - block = ((char*)CNBuffer) + blockStart; - dict = block - dictSize; - - /* Compression tests */ - - /* Test compression HC */ - FUZ_DISPLAYTEST; - ret = Lizard_compress(block, compressedBuffer, blockSize, Lizard_compressBound(blockSize), 0); - FUZ_CHECKTEST(ret==0, "Lizard_compress() failed"); - HCcompressedSize = ret; - - /* Test compression HC using external state */ - FUZ_DISPLAYTEST; - ret = Lizard_compress_extState(stateLizardHC, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize), 0); - FUZ_CHECKTEST(ret==0, "Lizard_compress_extState() failed"); - - /* Test compression using external state */ - FUZ_DISPLAYTEST; - ret = Lizard_compress_extState_MinLevel(stateLizard, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); - FUZ_CHECKTEST(ret==0, "Lizard_compress_extState_MinLevel(1) failed"); - - /* Test compression */ - FUZ_DISPLAYTEST; - ret = Lizard_compress_MinLevel(block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); - FUZ_CHECKTEST(ret==0, "Lizard_compress_MinLevel() failed"); - compressedSize = ret; - - /* Decompression tests */ - - crcOrig = XXH32(block, blockSize, 0); - - /* Test decoding with output size exactly what's necessary => must work */ - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = Lizard_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); - FUZ_CHECKTEST(ret<0, "Lizard_decompress_safe failed despite sufficient space"); - FUZ_CHECKTEST(ret!=blockSize, "Lizard_decompress_safe did not regenerate original data"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "Lizard_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "Lizard_decompress_safe corrupted decoded data"); - - // Test decoding with more than enough output size => must work - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - decodedBuffer[blockSize+1] = 0; - ret = Lizard_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1); - FUZ_CHECKTEST(ret<0, "Lizard_decompress_safe failed despite amply sufficient space"); - FUZ_CHECKTEST(ret!=blockSize, "Lizard_decompress_safe did not regenerate original data"); - //FUZ_CHECKTEST(decodedBuffer[blockSize], "Lizard_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ? - FUZ_CHECKTEST(decodedBuffer[blockSize+1], "Lizard_decompress_safe overrun specified output buffer size"); - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "Lizard_decompress_safe corrupted decoded data"); - - // Test decoding with output size being one byte too short => must fail - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = Lizard_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1); - FUZ_CHECKTEST(ret>=0, "Lizard_decompress_safe should have failed, due to Output Size being one byte too short"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "Lizard_decompress_safe overrun specified output buffer size"); - - // Test decoding with output size being 10 bytes too short => must fail - FUZ_DISPLAYTEST; - if (blockSize>10) - { - decodedBuffer[blockSize-10] = 0; - ret = Lizard_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10); - FUZ_CHECKTEST(ret>=0, "Lizard_decompress_safe should have failed, due to Output Size being 10 bytes too short"); - FUZ_CHECKTEST(decodedBuffer[blockSize-10], "Lizard_decompress_safe overrun specified output buffer size"); - } - - // Test decoding with input size being one byte too short => must fail - FUZ_DISPLAYTEST; - ret = Lizard_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize); - FUZ_CHECKTEST(ret>=0, "Lizard_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize); - - // Test decoding with input size being one byte too large => must fail - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - compressedBuffer[compressedSize] = 0; /* valgrind */ - ret = Lizard_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize); - FUZ_CHECKTEST(ret>=0, "Lizard_decompress_safe should have failed, due to input size being too large"); - FUZ_CHECKTEST(decodedBuffer[blockSize], "Lizard_decompress_safe overrun specified output buffer size"); - - // Test partial decoding with target output size being max/2 => must work - FUZ_DISPLAYTEST; - ret = Lizard_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize); - FUZ_CHECKTEST(ret<0, "Lizard_decompress_safe_partial failed despite sufficient space"); - - // Test partial decoding with target output size being just below max => must work - FUZ_DISPLAYTEST; - ret = Lizard_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize); - FUZ_CHECKTEST(ret<0, "Lizard_decompress_safe_partial failed despite sufficient space"); - - /* Test Compression with limited output size */ - - /* Test compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; - ret = Lizard_compress_MinLevel(block, compressedBuffer, blockSize, compressedSize); - FUZ_CHECKTEST(ret==0, "Lizard_compress_MinLevel() failed despite sufficient space"); - - /* Test compression with output size being exactly what's necessary and external state (should work) */ - FUZ_DISPLAYTEST; - ret = Lizard_compress_extState_MinLevel(stateLizard, block, compressedBuffer, blockSize, compressedSize); - FUZ_CHECKTEST(ret==0, "Lizard_compress_extState_MinLevel() failed despite sufficient space"); - - /* Test HC compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; - ret = Lizard_compress(block, compressedBuffer, blockSize, HCcompressedSize, 0); - FUZ_CHECKTEST(ret==0, "Lizard_compress() (limitedOutput) failed despite sufficient space"); - - /* Test HC compression with output size being exactly what's necessary (should work) */ - FUZ_DISPLAYTEST; - ret = Lizard_compress_extState(stateLizardHC, block, compressedBuffer, blockSize, HCcompressedSize, 0); - FUZ_CHECKTEST(ret==0, "Lizard_compress_extState() failed despite sufficient space"); - - /* Test compression with missing bytes into output buffer => must fail */ - FUZ_DISPLAYTEST; - { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; - if (missingBytes >= compressedSize) missingBytes = compressedSize-1; - missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ - compressedBuffer[compressedSize-missingBytes] = 0; - ret = Lizard_compress_MinLevel(block, compressedBuffer, blockSize, compressedSize-missingBytes); - FUZ_CHECKTEST(ret, "Lizard_compress_MinLevel should have failed (output buffer too small by %i byte)", missingBytes); - FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "Lizard_compress_MinLevel overran output buffer ! (%i missingBytes)", missingBytes) - } - - /* Test HC compression with missing bytes into output buffer => must fail */ - FUZ_DISPLAYTEST; - { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1; - if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1; - missingBytes += !missingBytes; /* avoid special case missingBytes==0 */ - compressedBuffer[HCcompressedSize-missingBytes] = 0; - ret = Lizard_compress(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, 0); - FUZ_CHECKTEST(ret, "Lizard_compress(limitedOutput) should have failed (output buffer too small by %i byte)", missingBytes); - FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "Lizard_compress overran output buffer ! (%i missingBytes)", missingBytes) - } - - - /*-******************/ - /* Dictionary tests */ - /*-******************/ - - /* Compress using dictionary */ - FUZ_DISPLAYTEST; - { Lizard_stream_t* Lizard_stream = Lizard_createStream_MinLevel(); - FUZ_CHECKTEST(Lizard_stream==NULL, "Lizard_createStream_MinLevel() allocation failed"); - Lizard_stream = Lizard_resetStream_MinLevel(Lizard_stream); - FUZ_CHECKTEST(Lizard_stream==NULL, "Lizard_resetStream_MinLevel() failed"); - Lizard_compress_continue (Lizard_stream, dict, compressedBuffer, dictSize, Lizard_compressBound(dictSize)); /* Just to fill hash tables */ - blockContinueCompressedSize = Lizard_compress_continue (Lizard_stream, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "Lizard_compress_continue failed"); - Lizard_freeStream(Lizard_stream); - } - - /* Compress using External dictionary */ - FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */ - if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - Lizard_loadDict(Lizarddict, dict, dictSize); - blockContinueCompressedSize = Lizard_compress_continue(Lizarddict, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "Lizard_compress_continue failed"); - - FUZ_DISPLAYTEST; - Lizard_loadDict(Lizarddict, dict, dictSize); - ret = Lizard_compress_continue(Lizarddict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "Lizard_compress_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); - - FUZ_DISPLAYTEST; - Lizard_loadDict(Lizarddict, dict, dictSize); - ret = Lizard_compress_continue(Lizarddict, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "Lizard_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "Lizard_compress_continue should work : enough size available within output buffer"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = Lizard_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "2Lizard_decompress_safe_usingDict did not regenerate original data ret[%d]!=blockSize[%d]", (int)ret, (int)blockSize); - FUZ_CHECKTEST(decodedBuffer[blockSize], "Lizard_decompress_safe_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "Lizard_decompress_safe_usingDict corrupted decoded data"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize-1] = 0; - ret = Lizard_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "Lizard_decompress_safe_usingDict should have failed : not enough output size (-1 byte)"); - FUZ_CHECKTEST(decodedBuffer[blockSize-1], "Lizard_decompress_safe_usingDict overrun specified output buffer size"); - - FUZ_DISPLAYTEST; - { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2; - if ((U32)blockSize > missingBytes) { - decodedBuffer[blockSize-missingBytes] = 0; - ret = Lizard_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "Lizard_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes); - FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "Lizard_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize); - } } - - /* Compress HC using External dictionary */ - FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ - if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; - Lizard_streamHCPtr = Lizard_resetStream (Lizard_streamHCPtr, FUZ_rand(&randState) & 0x7); - FUZ_CHECKTEST(Lizard_streamHCPtr==NULL, "Lizard_resetStream failed"); - Lizard_loadDict(Lizard_streamHCPtr, dict, dictSize); - blockContinueCompressedSize = Lizard_compress_continue(Lizard_streamHCPtr, block, compressedBuffer, blockSize, Lizard_compressBound(blockSize)); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "Lizard_compress_continue failed"); - - FUZ_DISPLAYTEST; - Lizard_loadDict(Lizard_streamHCPtr, dict, dictSize); - ret = Lizard_compress_continue(Lizard_streamHCPtr, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "Lizard_compress_continue using ExtDict should fail : one missing byte for output buffer"); - - FUZ_DISPLAYTEST; - Lizard_loadDict(Lizard_streamHCPtr, dict, dictSize); - ret = Lizard_compress_continue(Lizard_streamHCPtr, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "Lizard_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "Lizard_compress_continue should work : enough size available within output buffer"); - - FUZ_DISPLAYTEST; - decodedBuffer[blockSize] = 0; - ret = Lizard_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize); - FUZ_CHECKTEST(ret!=blockSize, "3Lizard_decompress_safe_usingDict did not regenerate original data ret[%d]!=blockSize[%d]", (int)ret, (int)blockSize); - FUZ_CHECKTEST(decodedBuffer[blockSize], "Lizard_decompress_safe_usingDict overrun specified output buffer size") - crcCheck = XXH32(decodedBuffer, blockSize, 0); - if (crcCheck!=crcOrig) - FUZ_findDiff(block, decodedBuffer); - FUZ_CHECKTEST(crcCheck!=crcOrig, "Lizard_decompress_safe_usingDict corrupted decoded data"); - - /* ***** End of tests *** */ - /* Fill stats */ - bytes += blockSize; - cbytes += compressedSize; - hcbytes += HCcompressedSize; - ccbytes += blockContinueCompressedSize; - } - - if (nbCycles<=1) nbCycles = cycleNb; /* end by time */ - bytes += !bytes; /* avoid division by 0 */ - printf("\r%7u /%7u - ", cycleNb, nbCycles); - printf("all tests completed successfully \n"); - printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); - printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); - printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100); - - /* release memory */ - { -_exit: - free(CNBuffer); - free(compressedBuffer); - free(decodedBuffer); - free(stateLizard); - free(stateLizardHC); - Lizard_freeStream(Lizard_streamHCPtr); - Lizard_freeStream(Lizarddict); - return result; - -_output_error: - result = 1; - goto _exit; - } -} - - -#define testInputSize (192 KB) -#define testCompressedSize (128 KB) -#define ringBufferSize (8 KB) - -static void FUZ_unitTests(U32 seed) -{ - const unsigned testNb = 0; - const unsigned cycleNb= 0; - char testInput[testInputSize]; - char testCompressed[testCompressedSize]; - char testVerify[testInputSize]; - char ringBuffer[ringBufferSize]; - U32 randState = seed ^ PRIME3; - - /* Init */ - FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState); - - /* 32-bits address space overflow test */ - FUZ_AddressOverflow(&randState); - - /* Lizard streaming tests */ - { Lizard_stream_t* statePtr; - Lizard_stream_t* streamingState; - U64 crcOrig; - U64 crcNew; - int result; - - /* Allocation test */ - statePtr = Lizard_createStream_MinLevel(); - FUZ_CHECKTEST(statePtr==NULL, "Lizard_createStream_MinLevel() allocation failed"); - Lizard_freeStream(statePtr); - - streamingState = Lizard_createStream_MinLevel(); - FUZ_CHECKTEST(streamingState==NULL, "Lizard_createStream_MinLevel() allocation failed"); - - /* simple compression test */ - crcOrig = XXH64(testInput, testCompressedSize, 0); - streamingState = Lizard_resetStream_MinLevel(streamingState); - FUZ_CHECKTEST(streamingState==NULL, "Lizard_resetStream_MinLevel() failed"); - result = Lizard_compress_continue(streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "Lizard_compress_continue() compression failed"); - - result = Lizard_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "Lizard_decompress_safe() decompression failed Level 1 (result=%d testCompressedSize=%d)", (int)result, (int)testCompressedSize); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "Lizard_decompress_safe() decompression corruption"); - - /* ring buffer test */ - { XXH64_state_t xxhOrig; - XXH64_state_t xxhNew; - Lizard_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 10; - const U32 maxMessageSizeMask = (1< ringBufferSize) rNext = 0; - if (dNext + messageSize > dBufferSize) dNext = 0; - } - } - Lizard_freeStream(streamingState); - } - - /* Lizard streaming tests */ - { Lizard_stream_t* streamPtr; - U64 crcOrig; - U64 crcNew; - int result; - - /* Allocation test */ - streamPtr = Lizard_createStream(0); - FUZ_CHECKTEST(streamPtr==NULL, "Lizard_createStream() allocation failed"); - - /* simple HC compression test */ - crcOrig = XXH64(testInput, testCompressedSize, 0); - streamPtr = Lizard_resetStream(streamPtr, 0); - FUZ_CHECKTEST(streamPtr==NULL, "Lizard_resetStream failed"); - result = Lizard_compress_continue(streamPtr, testInput, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "Lizard_compress_continue() compression failed"); - - result = Lizard_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "Lizard_decompress_safe() decompression failed Level 0 (result=%d testCompressedSize=%d)", (int)result, (int)testCompressedSize); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "Lizard_decompress_safe() decompression corruption"); - - /* simple dictionary HC compression test */ - crcOrig = XXH64(testInput + testInputSize - testCompressedSize, testCompressedSize, 0); - streamPtr = Lizard_resetStream(streamPtr, 0); - FUZ_CHECKTEST(streamPtr==NULL, "Lizard_resetStream failed"); - Lizard_loadDict(streamPtr, testInput, testInputSize - testCompressedSize); - result = Lizard_compress_continue(streamPtr, testInput + testInputSize - testCompressedSize, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "Lizard_compress_continue() dictionary compression failed : result = %i", result); - - result = Lizard_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, testInputSize - testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "Lizard_decompress_safe() simple dictionary decompression test failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "Lizard_decompress_safe() simple dictionary decompression test : corruption"); - - /* multiple HC compression test with dictionary */ - { int result1, result2; - int segSize = testCompressedSize / 2; - crcOrig = XXH64(testInput + segSize, testCompressedSize, 0); - streamPtr = Lizard_resetStream(streamPtr, 0); - FUZ_CHECKTEST(streamPtr==NULL, "Lizard_resetStream failed"); - Lizard_loadDict(streamPtr, testInput, segSize); - result1 = Lizard_compress_continue(streamPtr, testInput + segSize, testCompressed, segSize, segSize -1); - FUZ_CHECKTEST(result1==0, "Lizard_compress_continue() dictionary compression failed : result = %i", result1); - result2 = Lizard_compress_continue(streamPtr, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1); - FUZ_CHECKTEST(result2==0, "Lizard_compress_continue() dictionary compression failed : result = %i", result2); - - result = Lizard_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize); - FUZ_CHECKTEST(result!=segSize, "Lizard_decompress_safe() dictionary decompression part 1 failed"); - result = Lizard_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); - FUZ_CHECKTEST(result!=segSize, "Lizard_decompress_safe() dictionary decompression part 2 failed"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "Lizard_decompress_safe() dictionary decompression corruption"); - } - - /* remote dictionary HC compression test */ - crcOrig = XXH64(testInput + testInputSize - testCompressedSize, testCompressedSize, 0); - streamPtr = Lizard_resetStream(streamPtr, 0); - FUZ_CHECKTEST(streamPtr==NULL, "Lizard_resetStream failed"); - Lizard_loadDict(streamPtr, testInput, 32 KB); - result = Lizard_compress_continue(streamPtr, testInput + testInputSize - testCompressedSize, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "Lizard_compress_continue() remote dictionary failed : result = %i", result); - - result = Lizard_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "Lizard_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); - crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "Lizard_decompress_safe_usingDict() decompression corruption"); - - /* multiple HC compression with ext. dictionary */ - { XXH64_state_t crcOrigState; - XXH64_state_t crcNewState; - const char* dict = testInput + 3; - int dictSize = (FUZ_rand(&randState) & 8191); - char* dst = testVerify; - - size_t segStart = dictSize + 7; - int segSize = (FUZ_rand(&randState) & 8191); - int segNb = 1; - - streamPtr = Lizard_resetStream(streamPtr, 0); - FUZ_CHECKTEST(streamPtr==NULL, "Lizard_resetStream failed"); - Lizard_loadDict(streamPtr, dict, dictSize); - - XXH64_reset(&crcOrigState, 0); - XXH64_reset(&crcNewState, 0); - - while (segStart + segSize < testInputSize) { - XXH64_update(&crcOrigState, testInput + segStart, segSize); - crcOrig = XXH64_digest(&crcOrigState); - result = Lizard_compress_continue(streamPtr, testInput + segStart, testCompressed, segSize, Lizard_compressBound(segSize)); - FUZ_CHECKTEST(result==0, "Lizard_compress_continue() dictionary compression failed : result = %i", result); - - result = Lizard_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize); - FUZ_CHECKTEST(result!=segSize, "Lizard_decompress_safe_usingDict() dictionary decompression part %i failed", segNb); - XXH64_update(&crcNewState, dst, segSize); - crcNew = XXH64_digest(&crcNewState); - if (crcOrig!=crcNew) { - size_t c=0; - while (dst[c] == testInput[segStart+c]) c++; - DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)segSize); - } - FUZ_CHECKTEST(crcOrig!=crcNew, "Lizard_decompress_safe_usingDict() part %i corruption", segNb); - - dict = dst; - //dict = testInput + segStart; - dictSize = segSize; - - dst += segSize + 1; - segNb ++; - - segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1; - segSize = (FUZ_rand(&randState) & 8191); - } - } - - /* ring buffer test */ - { XXH64_state_t xxhOrig; - XXH64_state_t xxhNew; - Lizard_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 10; - const U32 maxMessageSizeMask = (1< ringBufferSize) rNext = 0; - if (dNext + messageSize > dBufferSize) dNext = 0; - } - } - - /* small decoder-side ring buffer test */ - { XXH64_state_t xxhOrig; - XXH64_state_t xxhNew; - Lizard_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 12; - const U32 maxMessageSizeMask = (1< dBufferSize) dNext = 0; - } - } - - Lizard_freeStream(streamPtr); - } - - printf("All unit tests completed successfully \n"); - return; -_output_error: - exit(1); -} - - -static int FUZ_usage(char* programName) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [args]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS); - DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); - DISPLAY( " -s# : Select seed (default:prompt user)\n"); - DISPLAY( " -t# : Select starting test number (default:0)\n"); - DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); - DISPLAY( " -v : verbose\n"); - DISPLAY( " -p : pause at the end\n"); - DISPLAY( " -h : display help and exit\n"); - return 0; -} - - -int main(int argc, char** argv) -{ - U32 seed=0; - int seedset=0; - int argNb; - int nbTests = NB_ATTEMPTS; - int testNb = 0; - int proba = FUZ_COMPRESSIBILITY_DEFAULT; - int pause = 0; - char* programName = argv[0]; - U32 duration = 0; - - /* Check command line */ - for(argNb=1; argNb='0') && (*argument<='9')) { - nbTests *= 10; - nbTests += *argument - '0'; - argument++; - } - break; - - case 'T': - argument++; - nbTests = 0; duration = 0; - for (;;) { - switch(*argument) - { - case 'm': duration *= 60; argument++; continue; - case 's': - case 'n': argument++; continue; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': duration *= 10; duration += *argument++ - '0'; continue; - } - break; - } - break; - - case 's': - argument++; - seed=0; seedset=1; - while ((*argument>='0') && (*argument<='9')) { - seed *= 10; - seed += *argument - '0'; - argument++; - } - break; - - case 't': /* select starting test nb */ - argument++; - testNb=0; - while ((*argument>='0') && (*argument<='9')) { - testNb *= 10; - testNb += *argument - '0'; - argument++; - } - break; - - case 'P': /* change probability */ - argument++; - proba=0; - while ((*argument>='0') && (*argument<='9')) { - proba *= 10; - proba += *argument - '0'; - argument++; - } - if (proba<0) proba=0; - if (proba>100) proba=100; - break; - default: ; - } - } - } - } - - printf("Starting Lizard fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LIZARD_VERSION_STRING); - - if (!seedset) { - time_t const t = time(NULL); - U32 const h = XXH32(&t, sizeof(t), 1); - seed = h % 10000; - } - printf("Seed = %u\n", seed); - - if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba); - - if (testNb==0) FUZ_unitTests(seed); - - if (nbTests<=0) nbTests=1; - - { int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); - if (pause) { - DISPLAY("press enter ... \n"); - (void)getchar(); - } - return result; - } -} diff --git a/contrib/lizard/visual/.gitignore b/contrib/lizard/visual/.gitignore deleted file mode 100644 index 538cd1a21c8..00000000000 --- a/contrib/lizard/visual/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -# Visual C++ -.vs/ -*Copy -*.db -*.opensdf -*.sdf -*.suo -*.user -*.opendb - -VS2010/bin/ diff --git a/contrib/lizard/visual/README.md b/contrib/lizard/visual/README.md deleted file mode 100644 index 527e4be1874..00000000000 --- a/contrib/lizard/visual/README.md +++ /dev/null @@ -1,53 +0,0 @@ -Projects for various integrated development environments (IDE) -============================================================== - -#### Included projects - -The following projects are included with the lizard distribution: -- `VS2010` - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015) - - -#### How to compile lizard with Visual Studio - -1. Install Visual Studio e.g. VS 2015 Community Edition (it's free). -2. Download the latest version of lizard from https://github.com/inikep/lizard/releases -3. Decompress ZIP archive. -4. Go to decompressed directory then to `visual` then `VS2010` and open `lizard.sln` -5. Visual Studio will ask about converting VS2010 project to VS2015 and you should agree. -6. Change `Debug` to `Release` and if you have 64-bit Windows change also `Win32` to `x64`. -7. Press F7 on keyboard or select `BUILD` from the menu bar and choose `Build Solution`. -8. If compilation will be fine a compiled executable will be in `visual\VS2010\bin\x64_Release\lizard.exe` - - -#### Projects available within lizard.sln - -The Visual Studio solution file `lizard.sln` contains many projects that will be compiled to the -`visual\VS2010\bin\$(Platform)_$(Configuration)` directory. For example `lizard` set to `x64` and -`Release` will be compiled to `visual\VS2010\bin\x64_Release\lizard.exe`. The solution file contains the -following projects: - -- `lizard` : Command Line Utility, supporting gzip-like arguments -- `datagen` : Synthetic and parametrable data generator, for tests -- `frametest` : Test tool that checks lizard_frame integrity on target platform -- `fullbench` : Precisely measure speed for each lizard inner functions -- `fuzzer` : Test tool, to check lizard integrity on target platform -- `liblizard` : A static Lizard library compiled to `liblizard_static.lib` -- `liblizard-dll` : A dynamic Lizard library (DLL) compiled to `liblizard.dll` with the import library `liblizard.lib` -- `fullbench-dll` : The fullbench program compiled with the import library; the executable requires Lizard DLL - - -#### Using Lizard DLL with Microsoft Visual C++ project - -The header files `lib\lizard.h`, `lib\lizardhc.h`, `lib\lizard_frame.h` and the import library -`visual\VS2010\bin\$(Platform)_$(Configuration)\liblizard.lib` are required to compile a -project using Visual C++. - -1. The path to header files should be added to `Additional Include Directories` that can - be found in Project Properties of Visual Studio IDE in the `C/C++` Property Pages on the `General` page. -2. The import library has to be added to `Additional Dependencies` that can - be found in Project Properties in the `Linker` Property Pages on the `Input` page. - If one will provide only the name `liblizard.lib` without a full path to the library - then the directory has to be added to `Linker\General\Additional Library Directories`. - -The compiled executable will require Lizard DLL which is available at -`visual\VS2010\bin\$(Platform)_$(Configuration)\liblizard.dll`. diff --git a/contrib/lizard/visual/VS2010/datagen/datagen.vcxproj b/contrib/lizard/visual/VS2010/datagen/datagen.vcxproj deleted file mode 100644 index 7f7581d01b8..00000000000 --- a/contrib/lizard/visual/VS2010/datagen/datagen.vcxproj +++ /dev/null @@ -1,165 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {D745AE2F-596A-403A-9B91-81A8C6779243} - Win32Proj - datagen - $(SolutionDir)bin\$(Platform)_$(Configuration)\ - $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ - - - - Application - true - Unicode - - - Application - true - Unicode - - - Application - false - true - Unicode - - - Application - false - true - Unicode - - - - - - - - - - - - - - - - - - - true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - false - - - Console - true - - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize295252 %(AdditionalOptions) - - - Console - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - false - false - - - Console - true - true - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - false - /analyze:stacksize295252 %(AdditionalOptions) - - - Console - true - true - true - - - - - - - - - - - - - \ No newline at end of file diff --git a/contrib/lizard/visual/VS2010/frametest/frametest.vcxproj b/contrib/lizard/visual/VS2010/frametest/frametest.vcxproj deleted file mode 100644 index b753d651be1..00000000000 --- a/contrib/lizard/visual/VS2010/frametest/frametest.vcxproj +++ /dev/null @@ -1,177 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7} - Win32Proj - frametest - $(SolutionDir)bin\$(Platform)_$(Configuration)\ - $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ - - - - Application - true - Unicode - - - Application - true - Unicode - - - Application - false - true - Unicode - - - Application - false - true - Unicode - - - - - - - - - - - - - - - - - - - true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - false - - - Console - true - - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize295252 %(AdditionalOptions) - - - Console - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - false - false - - - Console - true - true - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - false - true - /analyze:stacksize295252 %(AdditionalOptions) - - - Console - true - true - true - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/contrib/lizard/visual/VS2010/fullbench/fullbench.vcxproj b/contrib/lizard/visual/VS2010/fullbench/fullbench.vcxproj deleted file mode 100644 index f05bb298055..00000000000 --- a/contrib/lizard/visual/VS2010/fullbench/fullbench.vcxproj +++ /dev/null @@ -1,175 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E} - Win32Proj - fullbench - $(SolutionDir)bin\$(Platform)_$(Configuration)\ - $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ - - - - Application - true - Unicode - - - Application - true - Unicode - - - Application - false - true - Unicode - - - Application - false - true - Unicode - - - - - - - - - - - - - - - - - - - true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - false - - - Console - true - - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - false - - - Console - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - false - false - - - Console - true - true - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - false - false - - - Console - true - true - true - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/contrib/lizard/visual/VS2010/fuzzer/fuzzer.vcxproj b/contrib/lizard/visual/VS2010/fuzzer/fuzzer.vcxproj deleted file mode 100644 index 7fa0fec022f..00000000000 --- a/contrib/lizard/visual/VS2010/fuzzer/fuzzer.vcxproj +++ /dev/null @@ -1,174 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {18B9F1A7-9C66-4352-898B-30804DADE0FD} - Win32Proj - fuzzer - $(SolutionDir)bin\$(Platform)_$(Configuration)\ - $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ - - - - Application - true - Unicode - - - Application - true - Unicode - - - Application - false - true - Unicode - - - Application - false - true - Unicode - - - - - - - - - - - - - - - - - - - true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - false - - - Console - true - - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize295252 %(AdditionalOptions) - - - Console - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - false - false - - - Console - true - true - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - false - true - /analyze:stacksize295252 %(AdditionalOptions) - - - Console - true - true - true - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.rc b/contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.rc deleted file mode 100644 index 79110101f3b..00000000000 --- a/contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.rc +++ /dev/null @@ -1,51 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// - -#include "lizard_compress.h" /* LIZARD_VERSION_STRING */ -#define APSTUDIO_READONLY_SYMBOLS -#include "verrsrc.h" -#undef APSTUDIO_READONLY_SYMBOLS - - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -LANGUAGE 9, 1 - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION LIZARD_VERSION_MAJOR,LIZARD_VERSION_MINOR,LIZARD_VERSION_RELEASE,0 - PRODUCTVERSION LIZARD_VERSION_MAJOR,LIZARD_VERSION_MINOR,LIZARD_VERSION_RELEASE,0 - FILEFLAGSMASK VS_FFI_FILEFLAGSMASK -#ifdef _DEBUG - FILEFLAGS VS_FF_DEBUG -#else - FILEFLAGS 0x0L -#endif - FILEOS VOS_NT_WINDOWS32 - FILETYPE VFT_DLL - FILESUBTYPE VFT2_UNKNOWN -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904B0" - BEGIN - VALUE "CompanyName", "Yann Collet, Przemyslaw Skibinski" - VALUE "FileDescription", "Extremely fast compression" - VALUE "FileVersion", LIZARD_VERSION_STRING - VALUE "InternalName", "liblizard.dll" - VALUE "LegalCopyright", "Copyright (C) 2013-2017, Yann Collet, Przemyslaw Skibinski" - VALUE "OriginalFilename", "liblizard.dll" - VALUE "ProductName", "Lizard" - VALUE "ProductVersion", LIZARD_VERSION_STRING - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x0409, 1200 - END -END - -#endif diff --git a/contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.vcxproj b/contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.vcxproj deleted file mode 100644 index 09cdaa16180..00000000000 --- a/contrib/lizard/visual/VS2010/liblizard-dll/liblizard-dll.vcxproj +++ /dev/null @@ -1,179 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {00000000-3E71-41B3-BF68-4A7BDD8A5476} - Win32Proj - liblizard-dll - $(SolutionDir)bin\$(Platform)_$(Configuration)\ - $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ - - - - DynamicLibrary - true - Unicode - - - DynamicLibrary - true - Unicode - - - DynamicLibrary - false - true - Unicode - - - DynamicLibrary - false - true - Unicode - - - - - - - - - - - - - - - - - - - true - liblizard - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - true - liblizard - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - liblizard - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - false - liblizard - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - - - - Level4 - Disabled - WIN32;_DEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) - true - false - - - true - - - - - - - Level4 - Disabled - WIN32;_DEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) - true - true - /analyze:stacksize295252 %(AdditionalOptions) - - - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) - false - false - - - true - true - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) - false - true - /analyze:stacksize295252 %(AdditionalOptions) - - - true - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/contrib/lizard/visual/VS2010/liblizard/liblizard.vcxproj b/contrib/lizard/visual/VS2010/liblizard/liblizard.vcxproj deleted file mode 100644 index 15b74523b96..00000000000 --- a/contrib/lizard/visual/VS2010/liblizard/liblizard.vcxproj +++ /dev/null @@ -1,176 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476} - Win32Proj - liblizard - $(SolutionDir)bin\$(Platform)_$(Configuration)\ - $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ - - - - StaticLibrary - true - Unicode - - - StaticLibrary - true - Unicode - - - StaticLibrary - false - true - Unicode - - - StaticLibrary - false - true - Unicode - - - - - - - - - - - - - - - - - - - true - liblizard_static - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - true - liblizard_static - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - liblizard_static - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - false - liblizard_static - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - - - - Level4 - Disabled - WIN32;_DEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) - true - false - - - true - - - - - - - Level4 - Disabled - WIN32;_DEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) - true - true - /analyze:stacksize295252 %(AdditionalOptions) - - - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) - false - false - - - true - true - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;LIZARD_DLL_EXPORT=1;%(PreprocessorDefinitions) - false - true - /analyze:stacksize295252 %(AdditionalOptions) - - - true - true - true - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/contrib/lizard/visual/VS2010/lizard.sln b/contrib/lizard/visual/VS2010/lizard.sln deleted file mode 100644 index 72d68335c1e..00000000000 --- a/contrib/lizard/visual/VS2010/lizard.sln +++ /dev/null @@ -1,86 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Express 2012 for Windows Desktop -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lizard", "lizard\lizard.vcxproj", "{E30329AC-0057-4FE0-8FDA-7F650D398C4C}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblizard", "liblizard\liblizard.vcxproj", "{9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblizard-dll", "liblizard-dll\liblizard-dll.vcxproj", "{00000000-3E71-41B3-BF68-4A7BDD8A5476}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{18B9F1A7-9C66-4352-898B-30804DADE0FD}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "frametest", "frametest\frametest.vcxproj", "{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{D745AE2F-596A-403A-9B91-81A8C6779243}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - Release|Win32 = Release|Win32 - Release|x64 = Release|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.ActiveCfg = Debug|Win32 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.Build.0 = Debug|Win32 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.ActiveCfg = Debug|x64 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.Build.0 = Debug|x64 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.ActiveCfg = Release|Win32 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.Build.0 = Release|Win32 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.ActiveCfg = Release|x64 - {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.Build.0 = Release|x64 - {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.ActiveCfg = Debug|Win32 - {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.Build.0 = Debug|Win32 - {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.ActiveCfg = Debug|x64 - {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.Build.0 = Debug|x64 - {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.ActiveCfg = Release|Win32 - {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.Build.0 = Release|Win32 - {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.ActiveCfg = Release|x64 - {9092C5CC-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.Build.0 = Release|x64 - {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.ActiveCfg = Debug|Win32 - {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Debug|Win32.Build.0 = Debug|Win32 - {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.ActiveCfg = Debug|x64 - {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Debug|x64.Build.0 = Debug|x64 - {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.ActiveCfg = Release|Win32 - {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Release|Win32.Build.0 = Release|Win32 - {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.ActiveCfg = Release|x64 - {00000000-3E71-41B3-BF68-4A7BDD8A5476}.Release|x64.Build.0 = Release|x64 - {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.ActiveCfg = Debug|Win32 - {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.Build.0 = Debug|Win32 - {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.ActiveCfg = Debug|x64 - {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.Build.0 = Debug|x64 - {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.ActiveCfg = Release|Win32 - {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.Build.0 = Release|Win32 - {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.ActiveCfg = Release|x64 - {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.Build.0 = Release|x64 - {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.ActiveCfg = Debug|Win32 - {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.Build.0 = Debug|Win32 - {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.ActiveCfg = Debug|x64 - {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.Build.0 = Debug|x64 - {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.ActiveCfg = Release|Win32 - {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.Build.0 = Release|Win32 - {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.ActiveCfg = Release|x64 - {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.Build.0 = Release|x64 - {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.ActiveCfg = Debug|Win32 - {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.Build.0 = Debug|Win32 - {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.ActiveCfg = Debug|x64 - {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.Build.0 = Debug|x64 - {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.ActiveCfg = Release|Win32 - {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.Build.0 = Release|Win32 - {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.ActiveCfg = Release|x64 - {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.Build.0 = Release|x64 - {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.ActiveCfg = Debug|Win32 - {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.Build.0 = Debug|Win32 - {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.ActiveCfg = Debug|x64 - {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.Build.0 = Debug|x64 - {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.ActiveCfg = Release|Win32 - {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.Build.0 = Release|Win32 - {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.ActiveCfg = Release|x64 - {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/contrib/lizard/visual/VS2010/lizard/lizard.rc b/contrib/lizard/visual/VS2010/lizard/lizard.rc deleted file mode 100644 index 76c3c7f947e..00000000000 --- a/contrib/lizard/visual/VS2010/lizard/lizard.rc +++ /dev/null @@ -1,51 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// - -#include "lizard_compress.h" /* LIZARD_VERSION_STRING */ -#define APSTUDIO_READONLY_SYMBOLS -#include "verrsrc.h" -#undef APSTUDIO_READONLY_SYMBOLS - - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -LANGUAGE 9, 1 - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION LIZARD_VERSION_MAJOR,LIZARD_VERSION_MINOR,LIZARD_VERSION_RELEASE,0 - PRODUCTVERSION LIZARD_VERSION_MAJOR,LIZARD_VERSION_MINOR,LIZARD_VERSION_RELEASE,0 - FILEFLAGSMASK VS_FFI_FILEFLAGSMASK -#ifdef _DEBUG - FILEFLAGS VS_FF_DEBUG -#else - FILEFLAGS 0x0L -#endif - FILEOS VOS_NT_WINDOWS32 - FILETYPE VFT_DLL - FILESUBTYPE VFT2_UNKNOWN -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904B0" - BEGIN - VALUE "CompanyName", "Yann Collet, Przemyslaw Skibinski" - VALUE "FileDescription", "Extremely fast compression" - VALUE "FileVersion", LIZARD_VERSION_STRING - VALUE "InternalName", "lizard.exe" - VALUE "LegalCopyright", "Copyright (C) 2013-2017, Yann Collet, Przemyslaw Skibinski" - VALUE "OriginalFilename", "lizard.exe" - VALUE "ProductName", "Lizard" - VALUE "ProductVersion", LIZARD_VERSION_STRING - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x0409, 1200 - END -END - -#endif diff --git a/contrib/lizard/visual/VS2010/lizard/lizard.vcxproj b/contrib/lizard/visual/VS2010/lizard/lizard.vcxproj deleted file mode 100644 index 98be4d599db..00000000000 --- a/contrib/lizard/visual/VS2010/lizard/lizard.vcxproj +++ /dev/null @@ -1,190 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {E30329AC-0057-4FE0-8FDA-7F650D398C4C} - Win32Proj - lizard - $(SolutionDir)bin\$(Platform)_$(Configuration)\ - $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ - - - - Application - true - MultiByte - - - Application - true - MultiByte - - - Application - false - true - MultiByte - - - Application - false - true - MultiByte - - - - - - - - - - - - - - - - - - - true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - true - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - - - false - $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); - true - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - false - - - Console - true - setargv.obj;%(AdditionalDependencies) - - - - - - - Level4 - Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - true - true - /analyze:stacksize295252 %(AdditionalOptions) - - - Console - true - setargv.obj;%(AdditionalDependencies) - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - false - false - - - Console - true - setargv.obj;%(AdditionalDependencies) - true - true - - - - - Level4 - - - MaxSpeed - true - true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - false - true - /analyze:stacksize295252 %(AdditionalOptions) - - - Console - true - setargv.obj;%(AdditionalDependencies) - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/contrib/lzsse/.gitignore b/contrib/lzsse/.gitignore deleted file mode 100644 index 379a8f7f570..00000000000 --- a/contrib/lzsse/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -bin/* -gmake/* -vs2013/* -vs2015/* \ No newline at end of file diff --git a/contrib/lzsse/.travis.yml b/contrib/lzsse/.travis.yml deleted file mode 100644 index 68bb59ef0b9..00000000000 --- a/contrib/lzsse/.travis.yml +++ /dev/null @@ -1,185 +0,0 @@ -language: c -sudo: false -branches: - except: - - /^(wip\/)?(appveyor|msvc|mingw|windows)(\-.+)?$/ -matrix: - include: - ### - ## Linux builds using various versions of GCC. - ### - - env: C_COMPILER=gcc-7 - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - premake4 - packages: - - gcc-7 - - g++-7 - - premake4 - # - env: C_COMPILER=gcc-6 - # addons: - # apt: - # sources: - # - ubuntu-toolchain-r-test - # - premake4 - # packages: - # - gcc-6 - # - g++-6 - # - premake4 - - env: C_COMPILER=gcc-5 - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - premake4 - packages: - - gcc-5 - - g++-5 - - premake4 - # - env: C_COMPILER=gcc-4.9 - # addons: - # apt: - # sources: - # - ubuntu-toolchain-r-test - # - premake4 - # packages: - # - gcc-4.9 - # - g++-4.9 - # - premake4 - - env: C_COMPILER=gcc-4.8 - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - premake4 - packages: - - gcc-4.8 - - g++-4.8 - - premake4 - # - env: C_COMPILER=gcc-4.7 - # addons: - # apt: - # sources: - # - ubuntu-toolchain-r-test - # - premake4 - # packages: - # - gcc-4.7 - # - g++-4.7 - # - premake4 - - env: C_COMPILER=gcc-4.6 - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - premake4 - packages: - - gcc-4.6 - - g++-4.6 - - premake4 - # - os: linux - # env: C_COMPILER=gcc-4.5 - # addons: - # apt: - # sources: - # - ubuntu-toolchain-r-test - # - premake4 - # packages: - # - gcc-4.5 - # - g++-4.5 - # - premake4 - - env: C_COMPILER=gcc-4.4 - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - premake4 - packages: - - gcc-4.4 - - g++-4.4 - - premake4 - - ### - ## clang on Linux - ### - - env: C_COMPILER=clang-3.9 - addons: - apt: - sources: - - llvm-toolchain-precise-3.9 - - ubuntu-toolchain-r-test - - premake4 - packages: - - clang-3.9 - - premake4 - # - env: C_COMPILER=clang-3.8 - # addons: - # apt: - # sources: - # - llvm-toolchain-precise-3.8 - # - ubuntu-toolchain-r-test - # - premake4 - # packages: - # - clang-3.8 - # - premake4 - - env: C_COMPILER=clang-3.7 - addons: - apt: - sources: - - llvm-toolchain-precise-3.7 - - ubuntu-toolchain-r-test - - premake4 - packages: - - clang-3.7 - - premake4 - # - env: C_COMPILER=clang-3.6 - # addons: - # apt: - # sources: - # - llvm-toolchain-precise-3.6 - # - ubuntu-toolchain-r-test - # - premake4 - # packages: - # - clang-3.6 - # - premake4 - - env: C_COMPILER=clang-3.5 - addons: - apt: - sources: - - llvm-toolchain-precise-3.5 - - ubuntu-toolchain-r-test - - premake4 - packages: - - clang-3.5 - - premake4 - - ### - ## PGI - ### - - env: C_COMPILER=pgcc ENABLE_OPENMP=y - addons: - apt: - sources: - - premake4 - packages: - - premake4 - - ### - ## OS X - ### - # - os: osx - -before_install: -### -## If we use the matrix to set CC/CXX Travis, overwrites the values, -## so instead we use C/CXX_COMPILER, then copy the values to CC/CXX -## here (after Travis has set CC/CXX). -### -- if [ "${C_COMPILER}" = "pgcc" ]; then wget -q -O /dev/stdout 'https://raw.githubusercontent.com/nemequ/pgi-travis/master/install-pgi.sh' | /bin/sh; fi -- if [ -n "${C_COMPILER}" ]; then export CC="${C_COMPILER}"; fi -- premake4 gmake - -script: -- cd gmake && make - diff --git a/contrib/lzsse/LICENSE b/contrib/lzsse/LICENSE deleted file mode 100644 index d13dc62210f..00000000000 --- a/contrib/lzsse/LICENSE +++ /dev/null @@ -1,23 +0,0 @@ -Copyright (c) 2016, Conor Stokes -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/lzsse/README.md b/contrib/lzsse/README.md deleted file mode 100644 index 6b777eecb52..00000000000 --- a/contrib/lzsse/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# LZSSE -[LZSS](https://en.wikipedia.org/wiki/Lempel%E2%80%93Ziv%E2%80%93Storer%E2%80%93Szymanski) designed for a branchless SSE decompression implementation. - -Three variants: -- LZSSE2, for high compression files with small literal runs. -- LZSSE4, for a more balanced mix of literals and matches. -- LZSSE8, for lower compression data with longer runs of matches. - -All three variants have an optimal parser implementation, which uses a quite strong match finder (very similar to LzFind) combined with a Storer-Szymanski style parse. LZSSE4 and LZSSE8 have "fast" compressor implementations, which use a simple hash table based matching and a greedy parse. - -Currently LZSSE8 is the recommended variant to use in the general case, as it generally performs well in most cases (and you have the option of both optimal parse and fast compression). LZSSE2 is recommended if you are only using text, especially heavily compressible text, but is slow/doesn't compress as well on less compressible data and binaries. - -The code is approaching production readiness and LZSSE2 and LZSSE8 have received a reasonable amount of testing. - -See these blog posts [An LZ Codec Designed for SSE Decompression](http://conorstokes.github.io/compression/2016/02/15/an-LZ-codec-designed-for-SSE-decompression) and [Compressor Improvements and LZSSE2 vs LZSSE8](http://conorstokes.github.io/compression/2016/02/24/compressor-improvements-and-lzsse2-vs-lzsse8) for a description of how the compression algorithm and implementation function. There are also benchmarks, but these may not be upto date (in particular the figures in the initial blog post no longer represent compression performance). diff --git a/contrib/lzsse/example/main.cpp b/contrib/lzsse/example/main.cpp deleted file mode 100644 index 9287f683c7d..00000000000 --- a/contrib/lzsse/example/main.cpp +++ /dev/null @@ -1,389 +0,0 @@ -// LZSSE.cpp : Defines the entry point for the console application. -// -#define _CRT_SECURE_NO_WARNINGS 1 - -#include -#include -#include -#include -#include "../lzsse2/lzsse2.h" -#include "../lzsse4/lzsse4.h" -#include "../lzsse8/lzsse8.h" - -static const uint32_t MAGIC_NUMBER = 0x28F19732; - -void DisplayUsage() -{ - printf( "Usage:\n" ); - printf( " lzsse [args] input_file output_file\n" ); - printf( "\n" ); - printf( "Arguments:\n" ); - printf( " -2 Compress in lzsse2 mode (default)\n" ); - printf( " -4 Compress in lzsse4 mode\n" ); - printf( " -8 Compress in lzsse8 mode\n" ); - printf( " -f Optimal parse (default)\n" ); - printf( " -o Fast parse (not available for lzsse2)\n" ); - printf( " -d Decompress\n" ); - printf( " -lN Compression level for optimal parse, where N is 1 to 17 (default 16)\n" ); - printf( " -bN Block size in KiB, default 131,072\n" ); - printf( "\n" ); -} - -static size_t CompressorFastLZSSE4( LZSSE4_FastParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength, unsigned int ) -{ - return LZSSE4_CompressFast( state, input, inputLength, output, outputLength ); -} - -static size_t CompressorFastLZSSE8( LZSSE8_FastParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength, unsigned int ) -{ - return LZSSE8_CompressFast( state, input, inputLength, output, outputLength ); -} - -template -void Compress( FILE* inputFile, FILE* outputFile, uint64_t blockSize, uint8_t mode, unsigned int level, State* state, size_t (*compressor)( State*, const void*, size_t, void*, size_t, unsigned int ) ) -{ - if ( state == nullptr ) - { - printf( "Couldn't allocate parse state\n" ); - exit( 1 ); - } - - if ( fwrite( &MAGIC_NUMBER, sizeof( uint32_t ), 1, outputFile ) == 0 ) - { - printf( "Couldn't write magic number\n" ); - exit( 1 ); - } - - if ( fwrite( &mode, sizeof( uint8_t ), 1, outputFile ) == 0 ) - { - printf( "Couldn't write stream type\n" ); - exit( 1 ); - } - - if ( fwrite( &blockSize, sizeof( uint64_t ), 1, outputFile ) == 0 ) - { - printf( "Couldn't write block size\n" ); - exit( 1 ); - } - - size_t typedBlockSize = static_cast< size_t >( blockSize ); - uint8_t* inputBuffer = reinterpret_cast< uint8_t* >( malloc( typedBlockSize ) ); - uint8_t* outputBuffer = reinterpret_cast< uint8_t* >( malloc( typedBlockSize ) ); - - if ( inputBuffer == nullptr || outputBuffer == nullptr ) - { - printf( "Couldn't allocate buffer memory\n" ); - exit( 1 ); - } - - for ( ;; ) - { - size_t readSize = fread( inputBuffer, 1, blockSize, inputFile ); - - if ( readSize == 0 ) - { - break; - } - - size_t compressedSize = compressor( state, inputBuffer, readSize, outputBuffer, typedBlockSize, level ); - - if ( compressedSize == 0 ) - { - printf( "Compression function failed\n" ); - exit( 1 ); - } - - uint32_t compressedLength = static_cast< uint32_t >( compressedSize ); - uint32_t uncompressedLength = static_cast< uint32_t >( readSize ); - - if ( fwrite( &uncompressedLength, sizeof( uint32_t ), 1, outputFile ) < 1 ) - { - break; - } - - if ( fwrite( &compressedLength, sizeof( uint32_t ), 1, outputFile ) < 1 ) - { - printf( "Error writing compressed length from block\n" ); - exit( 1 ); - } - - if ( fwrite( outputBuffer, 1, compressedLength, outputFile ) != compressedLength ) - { - printf( "Error writing block\n" ); - exit( 1 ); - } - } - - free( inputBuffer ); - free( outputBuffer ); -} - -void Decompress( FILE* inputFile, FILE* outputFile ) -{ - uint32_t magicNumber; - uint64_t blockSize = 128 * 1024 * 1024; - - if ( fread( &magicNumber, sizeof( uint32_t ), 1, inputFile ) < 1 || magicNumber != MAGIC_NUMBER ) - { - printf( "Couldn't read magic number, or magic number incorrect\n" ); - exit( 1 ); - } - - uint8_t streamType; - - if ( fread( &streamType, sizeof( uint8_t ), 1, inputFile ) < 1 ) - { - printf( "Couldn't read stream type\n" ); - exit( 1 ); - } - - if ( fread( &blockSize, sizeof( uint64_t ), 1, inputFile ) < 1 ) - { - printf( "Couldn't read block size\n" ); - exit( 1 ); - } - - uint8_t* inputBuffer = reinterpret_cast< uint8_t* >( malloc( static_cast< size_t >( blockSize ) ) ); - uint8_t* outputBuffer = reinterpret_cast< uint8_t* >( malloc( static_cast< size_t >( blockSize ) ) ); - - if ( inputBuffer == nullptr || outputBuffer == nullptr ) - { - printf( "Couldn't allocate buffer memory\n" ); - exit( 1 ); - } - - size_t( *decompressor )( const void*, size_t, void*, size_t ); - - switch ( streamType ) - { - case 2: - - decompressor = LZSSE2_Decompress; - break; - - case 4: - - decompressor = LZSSE4_Decompress; - break; - - case 8: - - decompressor = LZSSE8_Decompress; - break; - - default: - - printf( "Invalid stream type\n" ); - exit( 1 ); - - } - - memset( inputBuffer, 0, blockSize ); - memset( outputBuffer, 0, blockSize ); - - for ( ;; ) - { - uint32_t compressedLength; - uint32_t uncompressedLength; - - if ( fread( &uncompressedLength, sizeof( uint32_t ), 1, inputFile ) < 1 ) - { - break; - } - - if ( fread( &compressedLength, sizeof( uint32_t ), 1, inputFile ) < 1 ) - { - printf( "Error reading compressed length from block\n" ); - exit( 1 ); - } - - if ( fread( inputBuffer, 1, compressedLength, inputFile ) != compressedLength ) - { - printf( "Error reading block\n" ); - exit( 1 ); - } - - size_t decompressedSize = 0; - - decompressedSize = - decompressor( inputBuffer, - compressedLength, - outputBuffer, - uncompressedLength ); - - if ( decompressedSize != size_t( uncompressedLength ) ) - { - printf( "Error in decompression stream\n" ); - exit( 1 ); - } - - if ( fwrite( outputBuffer, 1, uncompressedLength, outputFile ) != uncompressedLength ) - { - printf( "Couldn't write block to output file\n" ); - exit( 1 ); - } - } - - free( inputBuffer ); - free( outputBuffer ); -} - -int main( int argc, const char** argv ) -{ - bool decompression = false; - bool optimal = true; - uint64_t blockSize = 128 * 1024 * 1024; - uint8_t mode = 2; - unsigned int level = 16; - - if ( argc < 3 ) - { - DisplayUsage(); - exit( 1 ); - } - - for ( int argIndex = 1; argIndex < argc - 2; ++argIndex ) - { - const char* arg = argv[ argIndex ]; - - if ( arg[ 0 ] == '-' ) - { - switch ( arg[ 1 ] ) - { - case 'd': - - decompression = true; - break; - - case '2': - - mode = 2; - break; - - case '4': - - mode = 4; - break; - - case '8': - - mode = 8; - break; - - case 'l': - - level = static_cast< unsigned int >( strtoul( arg + 2, nullptr, 10 ) ); - break; - - case 'b': - - blockSize = strtoull( arg + 2, nullptr, 10 ) * 1024; - break; - - case 'o': - - optimal = true; - break; - - case 'f': - - optimal = false; - break; - - } - } - } - - FILE* inputFile = fopen( argv[ argc - 2 ], "rb" ); - - if ( inputFile == nullptr ) - { - perror( argv[ argc - 2 ] ); - exit( 1 ); - } - - FILE* outputFile = fopen( argv[ argc - 1 ], "wb+" ); - - if ( outputFile == nullptr ) - { - perror( argv[ argc - 2 ] ); - exit( 1 ); - } - - if ( decompression ) - { - Decompress( inputFile, outputFile ); - } - else - { - switch ( mode ) - { - case 2: - { - LZSSE2_OptimalParseState* state = LZSSE2_MakeOptimalParseState( static_cast< size_t >( blockSize ) ); - - Compress( inputFile, outputFile, blockSize, mode, level, state, LZSSE2_CompressOptimalParse ); - - LZSSE2_FreeOptimalParseState( state ); - - break; - } - - case 4: - { - if ( optimal ) - { - LZSSE4_OptimalParseState* state = LZSSE4_MakeOptimalParseState( static_cast( blockSize ) ); - - Compress( inputFile, outputFile, blockSize, mode, level, state, LZSSE4_CompressOptimalParse ); - - LZSSE4_FreeOptimalParseState( state ); - } - else - { - LZSSE4_FastParseState* state = LZSSE4_MakeFastParseState(); - - Compress( inputFile, outputFile, blockSize, mode, level, state, CompressorFastLZSSE4 ); - - LZSSE4_FreeFastParseState( state ); - } - - break; - } - - case 8: - { - if ( optimal ) - { - LZSSE8_OptimalParseState* state = LZSSE8_MakeOptimalParseState( static_cast( blockSize ) ); - - Compress( inputFile, outputFile, blockSize, mode, level, state, LZSSE8_CompressOptimalParse ); - - LZSSE8_FreeOptimalParseState( state ); - } - else - { - LZSSE8_FastParseState* state = LZSSE8_MakeFastParseState(); - - Compress( inputFile, outputFile, blockSize, mode, level, state, CompressorFastLZSSE8 ); - - LZSSE8_FreeFastParseState( state ); - } - - break; - } - - default: - - printf( "Invalid stream type\n" ); - exit( 1 ); - - } - } - - fclose( inputFile ); - fclose( outputFile ); - - return 0; -} - diff --git a/contrib/lzsse/lzsse2/lzsse2.cpp b/contrib/lzsse/lzsse2/lzsse2.cpp deleted file mode 100644 index d521a84553a..00000000000 --- a/contrib/lzsse/lzsse2/lzsse2.cpp +++ /dev/null @@ -1,1080 +0,0 @@ -/* -Copyright (c) 2016, Conor Stokes -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#include -#include -#include -#include - -#include "lzsse2_platform.h" -#include "lzsse2.h" - -#pragma warning ( disable : 4127 ) - -namespace -{ - // Constants - most of these should not be changed without corresponding code changes because it will break many things in unpredictable ways. - const uint32_t WINDOW_BITS = 16; - const uint32_t MIN_MATCH_LENGTH = 3; - const uint32_t LZ_WINDOW_SIZE = 1 << WINDOW_BITS; - const uint32_t LZ_WINDOW_MASK = LZ_WINDOW_SIZE - 1; - const uint32_t OPTIMAL_HASH_BITS = 20; - const uint32_t OPTIMAL_BUCKETS_COUNT = 1 << OPTIMAL_HASH_BITS; - const uint32_t OPTIMAL_HASH_MASK = OPTIMAL_BUCKETS_COUNT - 1; - const uint32_t MIN_COMPRESSION_SIZE = 32; - const uint32_t END_PADDING_LITERALS = 16; - const int32_t NO_MATCH = -1; - const int32_t EMPTY_NODE = -1; - const uint32_t MIN_LITERAL_COUNT = 8; - const uint32_t HASH_MULTIPLIER = 4013; - const uint32_t HASH_REMOVAL_MULTIPLIER = HASH_MULTIPLIER * HASH_MULTIPLIER; - const uint32_t CONTROL_BITS = 4; - const uint32_t LITERAL_BITS = 8; - const uint32_t OFFSET_BITS = 16; - const uint32_t BASE_MATCH_BITS = OFFSET_BITS + CONTROL_BITS; - const uint32_t SINGLE_LITERAL_COST = CONTROL_BITS + LITERAL_BITS; - const uint32_t DOUBLE_LITERAL_COST = SINGLE_LITERAL_COST + LITERAL_BITS; - const uint32_t EXTENDED_MATCH_BOUND = ( 1 << CONTROL_BITS ) - 1; - const uint32_t CONTROL_BLOCK_SIZE = sizeof( __m128i ); - const uint32_t CONTROLS_PER_BLOCK = 32; - const uint32_t LITERALS_PER_CONTROL = 2; - const uint32_t MAX_INPUT_PER_CONTROL = 2; - const size_t OUTPUT_BUFFER_SAFE = EXTENDED_MATCH_BOUND * CONTROLS_PER_BLOCK; - const size_t INPUT_BUFFER_SAFE = MAX_INPUT_PER_CONTROL * CONTROLS_PER_BLOCK; - const uint16_t INITIAL_OFFSET = MIN_MATCH_LENGTH; - const size_t SKIP_MATCH_LENGTH = 128; - const uint32_t NO_SKIP_LEVEL = 17; -} - -struct Arrival -{ - size_t cost; - int32_t from; - int32_t to; - uint16_t offset; -}; - -struct TreeNode -{ - int32_t children[ 2 ]; -}; - -struct LZSSE2_OptimalParseState -{ - // Note, we should really replace this with a BST, hash chaining works but is *slooooooooooooooow* for optimal parse. - int32_t roots[ OPTIMAL_BUCKETS_COUNT ]; - - TreeNode window[ LZ_WINDOW_SIZE ]; - - Arrival* arrivals; - - size_t bufferSize; -}; - - -LZSSE2_OptimalParseState* LZSSE2_MakeOptimalParseState( size_t bufferSize ) -{ - if ( bufferSize > 0 && ( SIZE_MAX / sizeof( Arrival ) ) < bufferSize ) - { - return nullptr; - } - - LZSSE2_OptimalParseState* result = reinterpret_cast< LZSSE2_OptimalParseState* >( ::malloc( sizeof( LZSSE2_OptimalParseState ) ) ); - - result->bufferSize = bufferSize; - - if ( result != nullptr ) - { - result->arrivals = reinterpret_cast< Arrival* >( ::malloc( sizeof( Arrival ) * bufferSize ) ); - - if ( result->arrivals == nullptr ) - { - LZSSE2_FreeOptimalParseState( result ); - - result = nullptr; - } - } - - return result; -} - - -void LZSSE2_FreeOptimalParseState( LZSSE2_OptimalParseState* toFree ) -{ - ::free( toFree->arrivals ); - - toFree->arrivals = nullptr; - - ::free( toFree ); -} - - -inline uint32_t CalculateHash( const uint8_t* inputCursor ) -{ - return ( uint32_t( inputCursor[ 0 ] ) * HASH_MULTIPLIER * HASH_MULTIPLIER + uint32_t( inputCursor[ 1 ] ) * HASH_MULTIPLIER + uint32_t( inputCursor[ 2 ] ) ) & OPTIMAL_HASH_MASK; -} - - -struct Match -{ - size_t length; - int32_t position; - uint16_t offset; -}; - - -inline Match SearchAndUpdateFinder( LZSSE2_OptimalParseState& state, const uint8_t* input, const uint8_t* inputCursor, const uint8_t* inputEnd, uint32_t cutOff ) -{ - Match result; - - int32_t position = static_cast( inputCursor - input ); - - result.position = NO_MATCH; - result.length = MIN_MATCH_LENGTH; - result.offset = 0; - - size_t lengthToEnd = inputEnd - inputCursor; - int32_t lastPosition = position - ( LZ_WINDOW_SIZE - 1 ); - uint32_t hash = CalculateHash( inputCursor ); - - lastPosition = lastPosition > 0 ? lastPosition : 0; - - int32_t treeCursor = state.roots[ hash ]; - - state.roots[ hash ] = position; - - int32_t* left = &state.window[ position & LZ_WINDOW_MASK ].children[ 1 ]; - int32_t* right = &state.window[ position & LZ_WINDOW_MASK ].children[ 0 ]; - size_t leftLength = 0; - size_t rightLength = 0; - - for ( ;; ) - { - if ( cutOff-- == 0 || treeCursor < lastPosition ) - { - *left = *right = EMPTY_NODE; - break; - } - - TreeNode& currentNode = state.window[ treeCursor & LZ_WINDOW_MASK ]; - const uint8_t* key = input + treeCursor; - size_t matchLength = leftLength < rightLength ? leftLength : rightLength; - - uint16_t matchOffset = static_cast( position - treeCursor ); - size_t maxLength = matchOffset <= ( EXTENDED_MATCH_BOUND + 1 ) && matchOffset < lengthToEnd ? matchOffset : lengthToEnd; - - while ( matchLength < lengthToEnd ) - { - __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); - __m128i match16 = _mm_loadu_si128( reinterpret_cast( key + matchLength ) ); - - unsigned long matchBytes; - - _BitScanForward( &matchBytes, ( static_cast( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); - - matchLength += matchBytes; - - if ( matchBytes != 16 ) - { - break; - } - } - - matchLength = matchLength < lengthToEnd ? matchLength : lengthToEnd; - - size_t truncatedMatchLength = matchLength < maxLength ? matchLength : maxLength; - - if ( truncatedMatchLength >= result.length ) - { - result.length = truncatedMatchLength; - result.offset = matchOffset; - result.position = treeCursor; - } - - if ( matchLength == lengthToEnd ) - { - *left = currentNode.children[ 1 ]; - *right = currentNode.children[ 0 ]; - break; - } - - if ( inputCursor[ matchLength ] < key[ matchLength ] || ( matchLength == lengthToEnd ) ) - { - *left = treeCursor; - left = currentNode.children; - treeCursor = *left; - leftLength = matchLength; - } - else - { - *right = treeCursor; - right = currentNode.children + 1; - treeCursor = *right; - rightLength = matchLength; - } - } - - // Special RLE overlapping match case, the LzFind style match above doesn't work very well with our - // restriction of overlapping matches having offsets of at least 16. - // Suffix array seems like a better option to handling this. - { - // Note, we're detecting long RLE here, but if we have an offset too close, we'll sacrifice a fair - // amount of decompression performance to load-hit-stores. - int32_t matchPosition = position - ( sizeof( __m128i ) * 2 ); - - if ( matchPosition >= 0 ) - { - uint16_t matchOffset = static_cast( position - matchPosition ); - const uint8_t* key = input + matchPosition; - size_t matchLength = 0; - - while ( matchLength < lengthToEnd ) - { - __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); - __m128i match16 = _mm_loadu_si128( reinterpret_cast( key + matchLength ) ); - - unsigned long matchBytes; - - _BitScanForward( &matchBytes, ( static_cast( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); - - matchLength += matchBytes; - - if ( matchBytes != 16 ) - { - break; - } - - } - - matchLength = matchLength < lengthToEnd ? matchLength : lengthToEnd; - - if ( matchLength >= result.length ) - { - result.length = matchLength; - result.offset = matchOffset; - result.position = matchPosition; - } - } - } - - return result; -} - - -size_t LZSSE2_CompressOptimalParse( LZSSE2_OptimalParseState* state, const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength, unsigned int level ) -{ - if ( outputLength < inputLength || state->bufferSize < inputLength ) - { - // error case, output buffer not large enough. - return 0; - } - - const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); - uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); - - if ( inputLength < MIN_COMPRESSION_SIZE ) - { - memcpy( output, input, inputLength ); - - return inputLength; - } - - const uint8_t* inputCursor = input; - const uint8_t* inputEnd = input + inputLength; - Arrival* arrivalWatermark = state->arrivals; - Arrival* arrival = state->arrivals; - uint32_t cutOff = 1 << level; - - for ( int32_t* rootCursor = state->roots, *end = rootCursor + OPTIMAL_BUCKETS_COUNT; rootCursor < end; rootCursor += 4 ) - { - rootCursor[ 0 ] = EMPTY_NODE; - rootCursor[ 1 ] = EMPTY_NODE; - rootCursor[ 2 ] = EMPTY_NODE; - rootCursor[ 3 ] = EMPTY_NODE; - } - - for ( uint32_t where = 0; where < MIN_MATCH_LENGTH; ++where ) - { - /*Match dummy = */ SearchAndUpdateFinder( *state, input, inputCursor, inputEnd - END_PADDING_LITERALS, cutOff ); - - ++inputCursor; - } - - arrival->cost = LITERAL_BITS * MIN_MATCH_LENGTH; - arrival->from = -1; - arrival->offset = 0; - - // loop through each character and project forward the matches at that character to calculate the cheapest - // path of arrival for each individual character. - for ( const uint8_t* earlyEnd = inputEnd - END_PADDING_LITERALS; inputCursor < earlyEnd; ++inputCursor, ++arrival ) - { - uint32_t lengthToEnd = static_cast< uint32_t >( earlyEnd - inputCursor ); - int32_t currentPosition = static_cast< int32_t >( inputCursor - input ); - Arrival* literalFirst = arrival + 1; - Arrival* literalSecond = arrival + 2; - size_t arrivalCost = arrival->cost; - - // NOTE - we currently assume only 2 literals filled in here, because the minimum match length is 3. - // If we wanted to go with a higher minimum match length, we would need to fill in more literals before hand. - // Also, because there is a maximum of 2 literals per control block assumed. - - // project forward the cost of a single literal - if ( literalFirst > arrivalWatermark || literalFirst->cost > ( arrival->cost + SINGLE_LITERAL_COST ) ) - { - literalFirst->cost = arrival->cost + SINGLE_LITERAL_COST; - literalFirst->from = currentPosition; - literalFirst->offset = 0; - - arrivalWatermark = literalFirst > arrivalWatermark ? literalFirst : arrivalWatermark; - } - - // project forward the cost of two literals - if ( lengthToEnd > 1 ) - { - if ( literalSecond > arrivalWatermark || literalFirst->cost > ( arrival->cost + DOUBLE_LITERAL_COST ) ) - { - literalSecond->cost = arrival->cost + DOUBLE_LITERAL_COST; - literalSecond->from = currentPosition; - literalSecond->offset = 0; - - arrivalWatermark = literalSecond > arrivalWatermark ? literalSecond : arrivalWatermark; - } - } - else - { - continue; - } - - Match match = SearchAndUpdateFinder( *state, input, inputCursor, earlyEnd, cutOff ); - - if ( match.position != NO_MATCH ) - { - for ( size_t matchedLength = MIN_MATCH_LENGTH, end = match.length + 1; matchedLength < end; ++matchedLength ) - { - Arrival* matchArrival = arrival + matchedLength; - size_t matchCost = arrivalCost + BASE_MATCH_BITS; - - if ( matchedLength > EXTENDED_MATCH_BOUND ) - { - matchCost += ( ( matchedLength - 1 ) / EXTENDED_MATCH_BOUND ) * CONTROL_BITS; - } - - if ( matchArrival > arrivalWatermark || matchArrival->cost > matchCost ) - { - matchArrival->cost = matchCost; - matchArrival->from = currentPosition; - matchArrival->offset = match.offset; - - arrivalWatermark = matchArrival > arrivalWatermark ? matchArrival : arrivalWatermark; - } - } - - if ( match.length > SKIP_MATCH_LENGTH && level < NO_SKIP_LEVEL ) - { - arrival += match.length - LITERALS_PER_CONTROL; - inputCursor += match.length - LITERALS_PER_CONTROL; - } - } - } - - // If this would cost more to encode than it would if it were just literals, encode it with no control blocks, - // just literals - if ( ( arrivalWatermark->cost + END_PADDING_LITERALS * LITERAL_BITS + CONTROLS_PER_BLOCK * CONTROL_BITS ) > ( inputLength * LITERAL_BITS ) ) - { - memcpy( output, input, inputLength ); - - return inputLength; - } - - Arrival* previousPathNode; - - // now trace the actual optimal parse path back, connecting the nodes in the other direction. - for ( const Arrival* pathNode = arrivalWatermark; pathNode->from > 0; pathNode = previousPathNode ) - { - previousPathNode = state->arrivals + ( pathNode->from - MIN_MATCH_LENGTH ); - - previousPathNode->to = static_cast( ( pathNode - state->arrivals ) + MIN_MATCH_LENGTH ); - } - - uint8_t* outputCursor = output; - - memcpy( outputCursor, input, MIN_MATCH_LENGTH ); - - outputCursor += MIN_MATCH_LENGTH; - - uint8_t* currentControlBlock = outputCursor; - uint32_t currentControlCount = 0; - uint32_t totalControlCount = 0; - - outputCursor += CONTROL_BLOCK_SIZE; - - Arrival* nextPathNode; - - size_t totalPathLength = MIN_MATCH_LENGTH; - uint16_t previousOffset = INITIAL_OFFSET; - - bool lastControlIsNop = false; - - // Now walk forwards again and actually write out the data. - for ( const Arrival* pathNode = state->arrivals; pathNode < arrivalWatermark; pathNode = nextPathNode ) - { - int32_t currentPosition = static_cast< int32_t >( ( pathNode - state->arrivals ) + MIN_MATCH_LENGTH ); - - nextPathNode = state->arrivals + ( pathNode->to - MIN_MATCH_LENGTH ); - - size_t pathDistance = nextPathNode - pathNode; - - totalPathLength += pathDistance; - - lastControlIsNop = false; - - if ( pathDistance < MIN_MATCH_LENGTH ) - { - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - currentControlCount = 0; - } - - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - ( static_cast( pathDistance ) - 1 ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - ( static_cast< uint8_t >( pathDistance ) - 1 ) << CONTROL_BITS; - } - - // output the literals. - for ( int32_t where = 0; where < pathDistance; ++where ) - { - const uint8_t* currentInput = input + currentPosition + where; - - outputCursor[ where ] = *currentInput ^ *( currentInput - previousOffset ); - } - - outputCursor += pathDistance; - - ++totalControlCount; - ++currentControlCount; - } - else - { - size_t toEncode = pathDistance - 1; //note, we always subtract one here, because the first control block of the match encodes this way - - // make sure the control block for the first part of the match has been allocated - // note, this is idempontent if we have not actually incremented the control count and we try this again. - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - currentControlCount = 0; - } - - // output the offset (after control block containing the first control nibble for this match). - *reinterpret_cast< uint16_t* >( outputCursor ) = nextPathNode->offset ^ previousOffset; - - previousOffset = nextPathNode->offset; - - outputCursor += sizeof( uint16_t ); - - for ( ;; ) - { - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - currentControlCount = 0; - } - - if ( toEncode >= EXTENDED_MATCH_BOUND ) - { - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - static_cast( EXTENDED_MATCH_BOUND ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - static_cast< uint8_t >( EXTENDED_MATCH_BOUND ) << CONTROL_BITS; - } - - toEncode -= EXTENDED_MATCH_BOUND; - - ++totalControlCount; - ++currentControlCount; - - } - else - { - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - static_cast( toEncode ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - static_cast< uint8_t >( toEncode ) << CONTROL_BITS; - } - - if ( toEncode == 0 && currentControlCount == 0 ) - { - lastControlIsNop = true; - } - - ++totalControlCount; - ++currentControlCount; - - break; - } - } - } - } - - if ( lastControlIsNop ) - { - outputCursor -= CONTROL_BLOCK_SIZE; - } - - size_t remainingLiterals = ( input + inputLength ) - inputCursor; - - // copy remaining literals - memcpy( outputCursor, inputCursor, remainingLiterals ); - - outputCursor += remainingLiterals; - - return outputCursor - output; -} - - -size_t LZSSE2_Decompress( const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength ) -{ - const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); - uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); - - // Length it not work compressing, just copy initial values - if ( outputLength == inputLength ) - { - memcpy( output, input, outputLength ); - - return inputLength; - } - - const uint8_t* inputCursor = input; - uint8_t* outputCursor = output; - - // The offset starts off as the minimum match length. We actually need it at least two - // characters back because we need them to be set to xor out the literals from the match data. - size_t offset = INITIAL_OFFSET; - __m128i previousCarryHi = _mm_setzero_si128(); - - *( outputCursor++ ) = *( inputCursor++ ); - *( outputCursor++ ) = *( inputCursor++ ); - *( outputCursor++ ) = *( inputCursor++ ); - - // What these macros do: - // Decode a single literal run or match run for a single control nibble. - // How they do it: - // - Read the *unaligned* input (in the case of LZSSE-F - twice, for LZSSE-O we read once) - one goes into an SSE register, - // because it could either be literals or an offset (or nothing at all). The low byte of streamBytesRead controls how much we advance - // the input cursor. - // - Used a contived set of casts to sign extend the "read offset" control mask and then use it to mask the input word, - // which is then xor'd against the offset, for a "branchless" conditional move into the offset which - // has been carried over from the previous literal/match block. Note, this ends up doing better than a cmov on most - // modern processors. But we need to pre-xor the input offset. - // - We then load the match data from output buffer (offset back from the current output point). Unconditional load here. - // - We broadcast the "from literal" control mask from the current least significant byte of the SSE register using a shuffle epi-8 - // - We mask the literals with that SSE register wide mask. - // - The literals have been pre-xor'd with the data read in as match data, so we use an xor to branchlessly choose between the two. - // In this case, it ends up a better option than a blendv on most processors. - // - Store the block. We store all 16 bytes of the SSE register (due to some constraints in the format of the data, we won't - // go past the end of the buffer), but we may overlap this. - // - bytesOut controls how much we advance the output cursor. - // - We use 8 bit shifts to advance all the controls up to the next byte. There is some variable sized register trickery that - // x86/x64 is great for as long as we don't anger the register renamer. - -#define DECODE_STEP( HILO, CHECKMATCH, CHECKBUFFERS ) \ - { \ - \ - uint64_t inputWord = *reinterpret_cast( inputCursor ); \ - __m128i literals = _mm_cvtsi64_si128( inputWord ); \ - \ - offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ - \ - readOffsetHalf##HILO >>= 8; \ - \ - const uint8_t* matchPointer = outputCursor - offset; \ - \ - if ( CHECKMATCH && matchPointer < output ) \ - goto MATCH_UNDERFLOW; \ - \ - __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ - __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ - \ - literals = _mm_and_si128( fromLiteral, literals ); \ - \ - fromLiteral##HILO = _mm_srli_si128( fromLiteral##HILO, 1 ); \ - \ - __m128i toStore = _mm_xor_si128( matchData, literals ); \ - \ - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ - \ - outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ - inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ - \ - bytesOutHalf##HILO >>= 8; \ - streamBytesReadHalf##HILO >>= 8; \ - \ - if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ - goto BUFFER_END; \ - } - -#define DECODE_STEP_HALF( HILO, CHECKMATCH, CHECKBUFFERS ) \ - { \ - size_t inputWord = *reinterpret_cast< const uint16_t* >( inputCursor ); \ - __m128i literals = _mm_cvtsi64_si128( inputWord ); \ - \ - offset ^= static_cast< size_t >( static_cast< ptrdiff_t >( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ - \ - const uint8_t* matchPointer = outputCursor - offset; \ - \ - if ( CHECKMATCH && matchPointer < output ) \ - goto MATCH_UNDERFLOW; \ - \ - __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ - __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ - \ - literals = _mm_and_si128( fromLiteral, literals ); \ - \ - fromLiteral##HILO = _mm_srli_si128( fromLiteral##HILO, 1 ); \ - \ - __m128i toStore = _mm_xor_si128( matchData, literals ); \ - \ - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ - \ - outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ - inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ - \ - if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ - goto BUFFER_END; \ - } - -#define DECODE_STEP_END( HILO, CHECKMATCH, CHECKBUFFERS ) \ - { \ - size_t inputWord = *reinterpret_cast< const uint16_t* >( inputCursor ); \ - __m128i literals = _mm_cvtsi64_si128( inputWord ); \ - \ - offset ^= static_cast< size_t >( static_cast< ptrdiff_t >( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ - \ - const uint8_t* matchPointer = outputCursor - offset; \ - \ - if ( CHECKMATCH && matchPointer < output ) \ - goto MATCH_UNDERFLOW; \ - \ - __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ - __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ - \ - literals = _mm_and_si128( fromLiteral, literals ); \ - \ - __m128i toStore = _mm_xor_si128( matchData, literals ); \ - \ - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ - \ - outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ - inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ - \ - if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ - goto BUFFER_END; \ - } - -#define DECODE_STEP_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP( Lo, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP( Hi, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_HALF_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_HALF( Lo, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_HALF_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_HALF( Hi, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_END_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_END( Lo, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_END_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_END( Hi, CHECKMATCH, CHECKBUFFERS ) - - __m128i nibbleMask = _mm_set1_epi8( 0xF ); - __m128i literalsPerControl = _mm_set1_epi8( LITERALS_PER_CONTROL ); - - // Note, we use this block here because it allows the "fake" inputEarlyEnd/outputEarlyEnd not to cause register spills - // in the decompression loops. And yes, that did actually happen. - { -#pragma warning ( push ) -#pragma warning ( disable : 4101 ) - - const uint8_t* inputEarlyEnd; //= ( input + inputLength ) - END_PADDING_LITERALS; - uint8_t* outputEarlyEnd;// = ( output + outputLength ) - END_PADDING_LITERALS; - -#pragma warning ( pop ) - - // "Safe" ends to the buffer, before the input/output cursors hit these, we can loop without overflow checks. - const uint8_t* inputSafeEnd = ( input + inputLength ) - INPUT_BUFFER_SAFE; - uint8_t* outputSafeEnd = ( output + outputLength ) - OUTPUT_BUFFER_SAFE; - - // Decoding loop with offset output buffer underflow test, but no buffer overflow tests, assumed to end at a safe distance - // from overflows - while ( ( outputCursor - output ) < LZ_WINDOW_SIZE && outputCursor < outputSafeEnd && inputCursor < inputSafeEnd ) - { - // load the control block - __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); - - // split the control block into high and low nibbles. - __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); - __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); - - // Here we are testing if the runs will be literals or matches. Note that if the carries are set from the previous operation - // this will essentially be ignored later on. - __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); - __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); - - // Work out the carry for the low nibbles (which will be used with the high controls to put them into - // match without offset read mode). - __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); - - // The carry for the high nibbles is used with the low controls, but needs one byte from the previous iteration. We save - // the calculated carry to use that byte next iteration. - __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); - __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); - - previousCarryHi = carryHi; - - // I want 128 set bits please. - __m128i allSet = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); - - // Calcualting the bytes to output to the stream. Basically, we are subtracting negative one from the control value if the - // carry is not set. This works because the masks produced by comparisons are the equivalent to negative one, which - // make this a conditional increment. - __m128i bytesOutLo = _mm_sub_epi8( controlLo, _mm_xor_si128( shiftedCarryHi, allSet ) ); - __m128i bytesOutHi = _mm_sub_epi8( controlHi, _mm_xor_si128( carryLo, allSet ) ); - - // Calculate the number of bytes to read per control. - // In the case the carry is set, no bytes. Otherwise, the offset size (2 bytes) for matches or the number of output bytes for literals. - __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_min_epi8( literalsPerControl, bytesOutLo ) ); - __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_min_epi8( literalsPerControl, bytesOutHi ) ); - - // Masks to read the offset (or keep the previous one) - set in the case that this is not a literal and the carry is not set - __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), allSet ); - __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), allSet ); - - // Masks whether we are reading literals - set if the carry is not set and these are literals. - __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); - __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); - - // Advance the input past the control block. - inputCursor += CONTROL_BLOCK_SIZE; - - { - // Pull out the bottom halves off the SSE registers from before - we want these - // things in GPRs for the more linear logic. - uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_HALF_LO( true, false ); - DECODE_STEP_HALF_HI( true, false ); - } - - { - // Now the top halves. - uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_END_LO( true, false ); - DECODE_STEP_END_HI( true, false ); - } - } - - // Decoding loop with no buffer checks, but will end at a safe distance from the end of the buffers. - // Note, when we get here we have already reached the point in the output buffer which is *past* where we can underflow - // due to a bad match offset. - while ( outputCursor < outputSafeEnd && inputCursor < inputSafeEnd ) - { - // This code is the same as the loop above, see comments there - __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); - __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); - __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); - - __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); - __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); - __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); - __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); - __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); // where we take the carry from the previous hi values - - previousCarryHi = carryHi; - - __m128i neg1 = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); - - __m128i bytesOutLo = _mm_sub_epi8( controlLo, _mm_xor_si128( shiftedCarryHi, neg1 ) ); - __m128i bytesOutHi = _mm_sub_epi8( controlHi, _mm_xor_si128( carryLo, neg1 ) ); - - __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_min_epi8( literalsPerControl, bytesOutLo ) ); - __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_min_epi8( literalsPerControl, bytesOutHi ) ); - - __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), neg1 ); - __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), neg1 ); - - __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); - __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); - - inputCursor += CONTROL_BLOCK_SIZE; - - { - uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_HALF_LO( false, false ); - DECODE_STEP_HALF_HI( false, false ); - } - - { - uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_END_LO( false, false ); - DECODE_STEP_END_HI( false, false ); - } - } - } - - // Decoding loop with all buffer checks. - { - const uint8_t* inputEarlyEnd; - uint8_t* outputEarlyEnd; - inputEarlyEnd = (( input + inputLength ) - END_PADDING_LITERALS); - outputEarlyEnd = ( output + outputLength ) - END_PADDING_LITERALS; - - while ( outputCursor < outputEarlyEnd && inputCursor < inputEarlyEnd ) - { - __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); - __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); - __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); - - __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); - __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); - __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); - __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); - __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); // where we take the carry from the previous hi values - - previousCarryHi = carryHi; - - __m128i neg1 = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); - - __m128i bytesOutLo = _mm_sub_epi8( controlLo, _mm_xor_si128( shiftedCarryHi, neg1 ) ); - __m128i bytesOutHi = _mm_sub_epi8( controlHi, _mm_xor_si128( carryLo, neg1 ) ); - - __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_min_epi8( literalsPerControl, bytesOutLo ) ); - __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_min_epi8( literalsPerControl, bytesOutHi ) ); - - __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), neg1 ); - __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), neg1 ); - - __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); - __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); - - inputCursor += CONTROL_BLOCK_SIZE; - - if ( inputCursor > inputEarlyEnd ) - goto BUFFER_END; - - { - uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_HALF_LO( true, true ); - DECODE_STEP_HALF_HI( true, true ); - } - - { - uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_END_LO( true, true ); - DECODE_STEP_END_HI( true, true ); - } - } - -BUFFER_END: - - // When we get here, we have either advanced the right amount on both cursors - // or something bad happened, so leave it as is, so we can tell where - // the error happened. - if ( inputCursor == inputEarlyEnd && outputCursor == outputEarlyEnd ) - { - size_t remainingLiterals = ( input + inputLength ) - inputCursor; - - // copy any trailing literals - memcpy( outputCursor, inputCursor, remainingLiterals ); - - outputCursor += remainingLiterals; - } - } - -MATCH_UNDERFLOW: - - return outputCursor - output; -} diff --git a/contrib/lzsse/lzsse2/lzsse2.h b/contrib/lzsse/lzsse2/lzsse2.h deleted file mode 100644 index ea908b85fb5..00000000000 --- a/contrib/lzsse/lzsse2/lzsse2.h +++ /dev/null @@ -1,90 +0,0 @@ -/* -Copyright (c) 2016, Conor Stokes -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef LZSSE2_H__ -#define LZSSE2_H__ - -#pragma once - -/* LZSSE2 - x64/SSE targeted codec for better performance with high compression ratio data/more optimal compressors. - * Supports minimum 3 byte matches, maximum 16 bytes of match per control word and 2 byte literal runs per control word. - */ - -#ifdef __cplusplus -extern "C" -{ -#endif - - /* Re-usable parse state object for compression. */ -typedef struct LZSSE2_OptimalParseState LZSSE2_OptimalParseState; - -/* Allocate the parse state for compression - returns null on failure. Note - Buffersize has to be greater or equal to any inputLength used with LZSSE2_CompressOptimalParse */ -LZSSE2_OptimalParseState* LZSSE2_MakeOptimalParseState( size_t bufferSize ); - -/* De-allocate the parse state for compression */ -void LZSSE2_FreeOptimalParseState( LZSSE2_OptimalParseState* toFree ); - -/* "Optimal" compression routine. -* Will compress data into LZSSE2 format, uses hash BST matching to find matches and run an optimal parse (high relative memory usage). Requires SSE 4.1. -* state : Contains the hash table for matching, passed as a parameter so that allocations can be re-used. -* input : Buffer containing uncompressed data to be compressed. May not be null. -* inputLength : Length of the compressed data in the input buffer - note should be under 2GB. -* output : Buffer that will receive the compressed output. -* outputLength : The length reserved in the buffer for compressed data. This should be at least inputLength. Note, -* The compressed data should never be longer than inputLength, as in this case the data is stored raw. -* level : The compression level to use for this file 1->17, 17 is highest compression, 0 is least -* Thread Safety - state can not be used on multiple threads with calls running concurrently. Can run multiple threads with separate state -* concurrently. -* -* Returns the size of the compressed data, or 0 in the case of error (e.g. outputLength is less than inputLength). -*/ -size_t LZSSE2_CompressOptimalParse( LZSSE2_OptimalParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength, unsigned int level ); - -/* Decompression routine. -* This routine will decompress data in the LZSSE2 format and currently requires SSE 4.1 and is targeted at x64. -* It will perform poorly on x86 due to hunger for registers. -* input : Buffer containing compressed input block. May not be null. -* inputLength : Length of the compressed data in the input buffer - note, this should be under 2GB -* output : Buffer that will received the de-compressed output. Note, that this needs to be at least outputLength long. -* May not be null. -* outputLength : The length of the compressed output - note, this should be under 2GB -* -* Provided that input and output are valid pointers to buffers of at least their specified size, this routine -* should be memory safe - both match pointer checks and input/output buffer checks exist. -* -* Returns the size of the decompressed data, which will be less than outputLength in the event of an error (number of bytes -* will indicate where in the output stream the error occured). -* -* Note that this data is not hash verified, errors that occur are either from a misformed stream or bad buffer sizes. -* Remember, corrupt data can still be valid to decompress. -*/ -size_t LZSSE2_Decompress( const void* input, size_t inputLength, void* output, size_t outputLength ); - -#ifdef __cplusplus -} -#endif - -#endif /* -- LZSSE2_H__ */ diff --git a/contrib/lzsse/lzsse2/lzsse2_platform.h b/contrib/lzsse/lzsse2/lzsse2_platform.h deleted file mode 100644 index 5cbab6b0600..00000000000 --- a/contrib/lzsse/lzsse2/lzsse2_platform.h +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright (c) 2016, Brian Marshall -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef LZSSE2_PLATFORM_H__ -#define LZSSE2_PLATFORM_H__ - -#pragma once - -/* - Compiler/Platform detection based on the table from: - https://blogs.msdn.microsoft.com/vcblog/2015/12/04/clang-with-microsoft-codegen-in-vs-2015-update-1/ -*/ - -#ifdef _MSC_VER - -/* - Microsoft Visual Studio Support. - C1xx/C2, Clang/C2 and Clang/LLVM all support the Microsoft header files and _BitScanForward - - Note: if you receive errors with the intrinsics make sure that you have SSE4.1 support enabled. - For example with Clang include "-msse4.1" on the command line -*/ -#include - -#else /* _MSC_VER */ - -#ifdef __GNUC__ - -/* - GCC -*/ - -/* - Note: including just would be sufficient, but including x86intrin is a better match to intrin.h on Visual Studio as - both include all intrinsics for the enabled processor, rather than just SSE4.1. -*/ -#include -/* _BitScanForward is Visual Studio specific. */ -#define _BitScanForward(x, m) *(x) = __builtin_ctz(m) - -#else - -/* -If you hit the error below, then add detection for your compiler/platform to this header file. -*/ -#error Platform not supported - -#endif /* __GNUC__ */ -#endif /* _MSC_VER */ - -#endif /* -- LZSSE2_PLATFORM_H__ */ diff --git a/contrib/lzsse/lzsse4/lzsse4.cpp b/contrib/lzsse/lzsse4/lzsse4.cpp deleted file mode 100644 index cf9f6fc6e60..00000000000 --- a/contrib/lzsse/lzsse4/lzsse4.cpp +++ /dev/null @@ -1,1499 +0,0 @@ -/* -Copyright (c) 2016, Conor Stokes -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#include -#include -#include -#include - -#include "lzsse4_platform.h" -#include "lzsse4.h" - -#pragma warning ( disable : 4127 ) - -namespace -{ - // Constants - most of these should not be changed without corresponding code changes because it will break many things in unpredictable ways. - const uint32_t WINDOW_BITS = 16; - const uint32_t MIN_MATCH_LENGTH = 4; - const uint32_t LZ_WINDOW_SIZE = 1 << WINDOW_BITS; - const uint32_t LZ_WINDOW_MASK = LZ_WINDOW_SIZE - 1; - const uint32_t FAST_HASH_BITS = 20; // You can change this - more bits = more matches, less bits = more cache hits - const uint32_t FAST_BUCKETS_COUNT = 1 << FAST_HASH_BITS; - const uint32_t FAST_HASH_MASK = FAST_BUCKETS_COUNT - 1; - const uint32_t MIN_COMPRESSION_SIZE = 32; - const uint32_t END_PADDING_LITERALS = 16; - const int32_t NO_MATCH = -1; - const int32_t EMPTY_NODE = -1; - const uint32_t MIN_LITERAL_COUNT = 8; - const uint32_t CONTROL_BITS = 4; - const uint32_t LITERAL_BITS = 8; - const uint32_t OFFSET_BITS = 16; - const uint32_t BASE_MATCH_BITS = OFFSET_BITS + CONTROL_BITS; - const uint32_t OFFSET_SIZE = 2; - const uint32_t EXTENDED_MATCH_BOUND = ( 1 << CONTROL_BITS ) - 1; - const uint32_t CONTROL_BLOCK_SIZE = sizeof( __m128i ); - const uint32_t CONTROLS_PER_BLOCK = 32; - const uint32_t LITERALS_PER_CONTROL = 4; - const uint32_t MAX_INPUT_PER_CONTROL = 4; - const size_t OUTPUT_BUFFER_SAFE = EXTENDED_MATCH_BOUND * CONTROLS_PER_BLOCK; - const size_t INPUT_BUFFER_SAFE = MAX_INPUT_PER_CONTROL * CONTROLS_PER_BLOCK; - const uint16_t INITIAL_OFFSET = MIN_MATCH_LENGTH; - const uint32_t OPTIMAL_HASH_BITS = 20; - const uint32_t OPTIMAL_BUCKETS_COUNT = 1 << OPTIMAL_HASH_BITS; - const uint32_t OPTIMAL_HASH_MASK = OPTIMAL_BUCKETS_COUNT - 1; - const size_t SKIP_MATCH_LENGTH = 128; - const uint32_t NO_SKIP_LEVEL = 17; -} - - -struct LZSSE4_FastParseState -{ - int32_t buckets[ FAST_BUCKETS_COUNT ]; // stores the first matching position, we can then look at the rest of the matches by tracing through the window. -}; - - -LZSSE4_FastParseState* LZSSE4_MakeFastParseState() -{ - return new LZSSE4_FastParseState(); -} - - -void LZSSE4_FreeFastParseState( LZSSE4_FastParseState* toFree ) -{ - delete toFree; -} - - -inline void SetHash( LZSSE4_FastParseState* state, uint32_t hash, const uint8_t* input, const uint8_t* inputCursor ) -{ - int32_t position = static_cast( inputCursor - input ); - - state->buckets[ hash & FAST_HASH_MASK ] = position; -} - -// Simple fast hash function - actually what is used in snappy and derivatives -// There's probably better, but I haven't spent time focusing on this area yet. -inline uint32_t HashFast( const uint8_t* inputCursor ) -{ - return *reinterpret_cast( inputCursor ) * 0x1e35a7bd >> ( 32 - FAST_HASH_BITS ); -} - -size_t LZSSE4_CompressFast( LZSSE4_FastParseState* state, const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength ) -{ - if ( outputLength < inputLength ) - { - // error case, output buffer not large enough. - return 0; - } - - const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); - uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); - - if ( inputLength < MIN_COMPRESSION_SIZE ) - { - memcpy( output, input, inputLength ); - - return inputLength; - } - - const uint8_t* inputCursor = input; - const uint8_t* inputEnd = input + inputLength; - const uint8_t* inputEarlyEnd = inputEnd - END_PADDING_LITERALS; - uint8_t* outputCursor = output; - uint8_t* outputEarlyEnd = ( output + outputLength ) - END_PADDING_LITERALS; - uint32_t hash = 0; - - // initialize hash to empty - for ( int32_t* where = state->buckets, *end = state->buckets + FAST_BUCKETS_COUNT; where < end; where += 4 ) - { - where[ 0 ] = -1; - where[ 1 ] = -1; - where[ 2 ] = -1; - where[ 3 ] = -1; - } - - // initial literals that wont be compressed - for ( uint32_t where = 0; where < MIN_MATCH_LENGTH; ++where ) - { - hash = HashFast( inputCursor ); - - SetHash( state, hash, input, inputCursor ); - - *( outputCursor++ ) = *( inputCursor++ ); - } - - uint8_t* currentControlBlock = outputCursor; - uint32_t currentControlCount = 0; - uint16_t previousOffset = INITIAL_OFFSET; - size_t literalsToFlush = 0; - - outputCursor += CONTROL_BLOCK_SIZE; - - bool lastControlIsNop = false; - - // Loop through the data until we hit the end of one of the buffers (minus the end padding literals) - while ( inputCursor < inputEarlyEnd && outputCursor <= outputEarlyEnd ) - { - lastControlIsNop = false; - - hash = HashFast( inputCursor ); - - int matchPosition = state->buckets[ hash & FAST_HASH_MASK ]; - int currentPosition = static_cast< int32_t >( inputCursor - input ); - uint32_t matchLength = 0; - uint16_t matchOffset = static_cast< uint16_t >( currentPosition - matchPosition ); - - // If we had a hit in the hash and it wasn't outside the window. - if ( matchPosition >= 0 && ( currentPosition - matchPosition ) < ( LZ_WINDOW_SIZE - 1 ) ) - { - const uint8_t* matchCandidate = input + matchPosition; - uint32_t lengthToEnd = static_cast< uint32_t >( inputEarlyEnd - inputCursor ); - // Here we limit the hash length to prevent overlap matches with offset less than 16 bytes - uint32_t maxLength = matchOffset <= ( EXTENDED_MATCH_BOUND + 1 ) && matchOffset < lengthToEnd ? matchOffset : lengthToEnd; - - // Find how long the match is 16 bytes at a time. - while ( matchLength < maxLength ) - { - __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); - __m128i match16 = _mm_loadu_si128( reinterpret_cast( matchCandidate + matchLength ) ); - - unsigned long matchBytes; - - // Finds the number of equal bytes at the start of the 16 - _BitScanForward( &matchBytes, ( static_cast< unsigned long >( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); - - matchLength += matchBytes; - - if ( matchBytes != sizeof( __m128i ) ) - { - break; - } - } - - matchLength = matchLength < maxLength ? matchLength : maxLength; - } - - // If we have at least the minimum match length (4 bytes) - if ( matchLength >= MIN_MATCH_LENGTH ) - { - // Do we have literals to flush before the match? - if ( literalsToFlush > 0 ) - { - // Start a new control block if we need one. - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - - // Would be larger than compressed size, get out! - if ( outputCursor > outputEarlyEnd ) - { - break; - } - } - - currentControlBlock[ currentControlCount >> 1 ] = - ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( literalsToFlush - 1 ) << 4 ); - - // flush the literals. - // note the xor against the data that would be read in the match. - for ( uint32_t where = 0; where < literalsToFlush; ++where ) - { - const uint8_t* currentInput = inputCursor - ( literalsToFlush - where ); - - *( outputCursor++ ) = *currentInput ^ *( currentInput - previousOffset ); - } - - ++currentControlCount; - - literalsToFlush = 0; - - // Would be larger than compressed size, get out! - if ( outputCursor > outputEarlyEnd ) - { - break; - } - } - - // Start a new control block if the previous one is full. - // Note this is done before the offset is written out - the offset - // is always written after the control block containing the first - // control in the match. - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - - if ( outputCursor > outputEarlyEnd ) - { - break; - } - } - - // The match length value we are encoding. - size_t toEncode = matchLength; - - // Write the offset out - note the xor with the previous offset. - *reinterpret_cast< uint16_t* >( outputCursor ) = matchOffset ^ previousOffset; - - previousOffset = matchOffset; - outputCursor += sizeof( uint16_t ); - - for ( ;; ) - { - // Check if we need to start a new control block - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - - if ( outputCursor > outputEarlyEnd ) - { - break; - } - } - - // If the encode size is greater than we can hold in a control, write out a full match length - // control, subtract full control value from the amount to encode and loop around again. - if ( toEncode >= EXTENDED_MATCH_BOUND ) - { - currentControlBlock[ currentControlCount >> 1 ] = - ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( EXTENDED_MATCH_BOUND ) << 4 ); - - toEncode -= EXTENDED_MATCH_BOUND; - - ++currentControlCount; - } - else // Write out the remaining match length control. Could potentially be zero. - { - currentControlBlock[ currentControlCount >> 1 ] = - ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( toEncode ) << 4 ); - - if ( currentControlCount == 0 && toEncode == 0 ) - { - lastControlIsNop = true; - } - - ++currentControlCount; - - break; - } - } - - // Update the value into the hash for future matches. - SetHash( state, hash, input, inputCursor ); - - ++inputCursor; - - // Hash all the other values in the match too. - for ( const uint8_t* nextArrival = inputCursor + matchLength - 1; inputCursor < nextArrival; ++inputCursor ) - { - hash = HashFast( inputCursor ); - SetHash( state, hash, input, inputCursor ); - } - } - else - { - // One more literal to write out. - ++literalsToFlush; - - // If we have reached the maximum number of literals allowed in the control, flush them out. - if ( literalsToFlush == LITERALS_PER_CONTROL ) - { - // Check if the control block is full and we need start a new one. - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - - if ( outputCursor > outputEarlyEnd ) - { - break; - } - } - - currentControlBlock[ currentControlCount >> 1 ] = - ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( ( static_cast( LITERALS_PER_CONTROL - 1 ) ) << 4 ); - - ++currentControlCount; - - *reinterpret_cast< uint32_t* >( outputCursor ) = - *reinterpret_cast< const uint32_t* >( inputCursor - 3 ) ^ - *reinterpret_cast< const uint32_t* >( ( inputCursor - 3 ) - previousOffset ); - - outputCursor += 4; - - //*( outputCursor++ ) = *( inputCursor - 3 ) ^ *( ( inputCursor - 3 ) - previousOffset ); - //*( outputCursor++ ) = *( inputCursor - 2 ) ^ *( ( inputCursor - 2 ) - previousOffset ); - //*( outputCursor++ ) = *( inputCursor - 1 ) ^ *( ( inputCursor - 1 ) - previousOffset ); - //*( outputCursor++ ) = *inputCursor ^ *( inputCursor - previousOffset ); - - if ( outputCursor > outputEarlyEnd ) - { - break; - } - - literalsToFlush = 0; - } - - // Update the hash with this byte - SetHash( state, hash, input, inputCursor ); - - ++inputCursor; - } - } - - // If we would create a compression output bigger than or equal to the input, just copy the input to the output and return equal size. - if ( ( ( outputCursor + literalsToFlush + ( currentControlCount == CONTROLS_PER_BLOCK ? CONTROL_BLOCK_SIZE : 0 ) ) ) >= output + inputLength - END_PADDING_LITERALS ) - { - memcpy( output, input, inputLength ); - - outputCursor = output + inputLength; - } - else - { - // Flush any remaining literals. - if ( literalsToFlush > 0 ) - { - lastControlIsNop = false; - - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - } - - currentControlBlock[ currentControlCount >> 1 ] = - ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( literalsToFlush - 1 ) << 4 ); - - for ( uint32_t where = 0; where < literalsToFlush; ++where ) - { - const uint8_t* currentInput = inputCursor - ( literalsToFlush - where ); - - *( outputCursor++ ) = *currentInput ^ *( currentInput - previousOffset ); - } - - ++currentControlCount; - } - - // Need to finish off shifting the final control block into the low nibble if there is no second nibble - if ( ( currentControlCount & 1 ) > 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] >>= 4; - } - - if ( lastControlIsNop ) - { - outputCursor -= CONTROL_BLOCK_SIZE; - } - - size_t remainingLiterals = ( input + inputLength ) - inputCursor; - - // copy remaining literals - memcpy( outputCursor, inputCursor, remainingLiterals ); - - outputCursor += remainingLiterals; - } - - // Return the size of the compressed data. - return outputCursor - output; -} - - -struct Arrival -{ - size_t cost; - int32_t from; - int32_t to; - uint16_t offset; -}; - -struct TreeNode -{ - int32_t children[ 2 ]; -}; - -struct LZSSE4_OptimalParseState -{ - // Note, we should really replace this with a BST, hash chaining works but is *slooooooooooooooow* for optimal parse. - int32_t roots[ OPTIMAL_BUCKETS_COUNT ]; - - TreeNode window[ LZ_WINDOW_SIZE ]; - - Arrival* arrivals; - - size_t bufferSize; -}; - - -LZSSE4_OptimalParseState* LZSSE4_MakeOptimalParseState( size_t bufferSize ) -{ - if ( bufferSize > 0 && ( SIZE_MAX / sizeof( Arrival ) ) < bufferSize ) - { - return nullptr; - } - - LZSSE4_OptimalParseState* result = reinterpret_cast< LZSSE4_OptimalParseState* >( ::malloc( sizeof( LZSSE4_OptimalParseState ) ) ); - - result->bufferSize = bufferSize; - - if ( result != nullptr ) - { - result->arrivals = reinterpret_cast< Arrival* >( ::malloc( sizeof( Arrival ) * bufferSize ) ); - - if ( result->arrivals == nullptr ) - { - LZSSE4_FreeOptimalParseState( result ); - - result = nullptr; - } - } - - return result; -} - - -void LZSSE4_FreeOptimalParseState( LZSSE4_OptimalParseState* toFree ) -{ - ::free( toFree->arrivals ); - - toFree->arrivals = nullptr; - - ::free( toFree ); -} - - -inline uint32_t HashOptimal( const uint8_t* inputCursor ) -{ - return *reinterpret_cast( inputCursor ) * 0x1e35a7bd >> ( 32 - OPTIMAL_HASH_BITS ); -} - - -struct Match -{ - size_t length; - int32_t position; - uint16_t offset; -}; - - -inline Match SearchAndUpdateFinder( LZSSE4_OptimalParseState& state, const uint8_t* input, const uint8_t* inputCursor, const uint8_t* inputEnd, uint32_t cutOff ) -{ - Match result; - - int32_t position = static_cast( inputCursor - input ); - - result.position = NO_MATCH; - result.length = MIN_MATCH_LENGTH; - result.offset = 0; - - size_t lengthToEnd = inputEnd - inputCursor; - int32_t lastPosition = position - ( LZ_WINDOW_SIZE - 1 ); - uint32_t hash = HashOptimal( inputCursor ); - - lastPosition = lastPosition > 0 ? lastPosition : 0; - - int32_t treeCursor = state.roots[ hash ]; - - state.roots[ hash ] = position; - - int32_t* left = &state.window[ position & LZ_WINDOW_MASK ].children[ 1 ]; - int32_t* right = &state.window[ position & LZ_WINDOW_MASK ].children[ 0 ]; - size_t leftLength = 0; - size_t rightLength = 0; - - for ( ;; ) - { - if ( cutOff-- == 0 || treeCursor < lastPosition ) - { - *left = *right = EMPTY_NODE; - break; - } - - TreeNode& currentNode = state.window[ treeCursor & LZ_WINDOW_MASK ]; - const uint8_t* key = input + treeCursor; - size_t matchLength = leftLength < rightLength ? leftLength : rightLength; - - uint16_t matchOffset = static_cast< uint16_t >( position - treeCursor ); - size_t maxLength = matchOffset <= ( EXTENDED_MATCH_BOUND + 1 ) && matchOffset < lengthToEnd ? matchOffset : lengthToEnd; - - while ( matchLength < lengthToEnd ) - { - __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); - __m128i match16 = _mm_loadu_si128( reinterpret_cast( key + matchLength ) ); - - unsigned long matchBytes; - - _BitScanForward( &matchBytes, ( static_cast( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); - - matchLength += matchBytes; - - if ( matchBytes != 16 ) - { - break; - } - } - - matchLength = matchLength < lengthToEnd ? matchLength : lengthToEnd; - - size_t truncatedMatchLength = matchLength < maxLength ? matchLength : maxLength; - - if ( truncatedMatchLength >= result.length && matchOffset >= LITERALS_PER_CONTROL ) - { - result.length = truncatedMatchLength; - result.offset = matchOffset; - result.position = treeCursor; - } - - if ( matchLength == lengthToEnd ) - { - *left = currentNode.children[ 1 ]; - *right = currentNode.children[ 0 ]; - break; - } - - if ( inputCursor[ matchLength ] < key[ matchLength ] || ( matchLength == lengthToEnd ) ) - { - *left = treeCursor; - left = currentNode.children; - treeCursor = *left; - leftLength = matchLength; - } - else - { - *right = treeCursor; - right = currentNode.children + 1; - treeCursor = *right; - rightLength = matchLength; - } - } - - // Special RLE overlapping match case, the LzFind style match above doesn't work very well with our - // restriction of overlapping matches having offsets of at least 16. - // Suffix array seems like a better option to handling this. - { - // Note, we're detecting long RLE here, but if we have an offset too close, we'll sacrifice a fair - // amount of decompression performance to load-hit-stores. - int32_t matchPosition = position - ( sizeof( __m128i ) * 2 ); - - if ( matchPosition >= 0 ) - { - uint16_t matchOffset = static_cast( position - matchPosition ); - const uint8_t* key = input + matchPosition; - size_t matchLength = 0; - - while ( matchLength < lengthToEnd ) - { - __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); - __m128i match16 = _mm_loadu_si128( reinterpret_cast( key + matchLength ) ); - - unsigned long matchBytes; - - _BitScanForward( &matchBytes, ( static_cast( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); - - matchLength += matchBytes; - - if ( matchBytes != 16 ) - { - break; - } - - } - - matchLength = matchLength < lengthToEnd ? matchLength : lengthToEnd; - - if ( matchLength >= result.length ) - { - result.length = matchLength; - result.offset = matchOffset; - result.position = matchPosition; - } - } - } - - return result; -} - - -size_t LZSSE4_CompressOptimalParse( LZSSE4_OptimalParseState* state, const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength, unsigned int level ) -{ - if ( outputLength < inputLength || state->bufferSize < inputLength ) - { - // error case, output buffer not large enough. - return 0; - } - - const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); - uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); - - if ( inputLength < MIN_COMPRESSION_SIZE ) - { - memcpy( output, input, inputLength ); - - return inputLength; - } - - const uint8_t* inputCursor = input; - const uint8_t* inputEnd = input + inputLength; - Arrival* arrivalWatermark = state->arrivals; - Arrival* arrival = state->arrivals; - uint32_t cutOff = 1 << level; - - for ( int32_t* rootCursor = state->roots, *end = rootCursor + OPTIMAL_BUCKETS_COUNT; rootCursor < end; rootCursor += 4 ) - { - rootCursor[ 0 ] = EMPTY_NODE; - rootCursor[ 1 ] = EMPTY_NODE; - rootCursor[ 2 ] = EMPTY_NODE; - rootCursor[ 3 ] = EMPTY_NODE; - } - - for ( uint32_t where = 0; where < MIN_MATCH_LENGTH; ++where ) - { - SearchAndUpdateFinder( *state, input, inputCursor, inputEnd - END_PADDING_LITERALS, cutOff ); - - ++inputCursor; - } - - arrival->cost = LITERAL_BITS * LITERALS_PER_CONTROL; - arrival->from = -1; - arrival->offset = 0; - - // loop through each character and project forward the matches at that character to calculate the cheapest - // path of arrival for each individual character. - for ( const uint8_t* earlyEnd = inputEnd - END_PADDING_LITERALS; inputCursor < earlyEnd; ++inputCursor, ++arrival ) - { - uint32_t lengthToEnd = static_cast< uint32_t >( earlyEnd - inputCursor ); - int32_t currentPosition = static_cast< int32_t >( inputCursor - input ); - size_t literalsForward = LITERALS_PER_CONTROL < lengthToEnd ? LITERALS_PER_CONTROL : lengthToEnd; - size_t arrivalCost = arrival->cost; - - // NOTE - we currently assume only 2 literals filled in here, because the minimum match length is 3. - // If we wanted to go with a higher minimum match length, we would need to fill in more literals before hand. - // Also, because there is a maximum of 2 literals per control block assumed. - - // project forward the cost of a single literal - - for ( size_t where = 1; where <= literalsForward; ++where ) - { - Arrival* literalArrival = arrival + where; - size_t literalCost = arrivalCost + CONTROL_BITS + ( where * LITERAL_BITS ); - - if ( literalArrival > arrivalWatermark || literalArrival->cost > literalCost ) - { - literalArrival->cost = literalCost; - literalArrival->from = currentPosition; - literalArrival->offset = 0; - - arrivalWatermark = literalArrival > arrivalWatermark ? literalArrival : arrivalWatermark; - } - } - - Match match = SearchAndUpdateFinder( *state, input, inputCursor, earlyEnd, cutOff ); - - if ( match.position != NO_MATCH ) - { - for ( size_t matchedLength = MIN_MATCH_LENGTH, end = match.length + 1; matchedLength < end; ++matchedLength ) - { - Arrival* matchArrival = arrival + matchedLength; - size_t matchCost = arrivalCost + BASE_MATCH_BITS; - - if ( matchedLength >= EXTENDED_MATCH_BOUND ) - { - matchCost += ( matchedLength / EXTENDED_MATCH_BOUND ) * CONTROL_BITS; - } - - if ( matchArrival > arrivalWatermark || matchArrival->cost > matchCost ) - { - matchArrival->cost = matchCost; - matchArrival->from = currentPosition; - matchArrival->offset = match.offset; - - arrivalWatermark = matchArrival > arrivalWatermark ? matchArrival : arrivalWatermark; - } - } - - if ( match.length > SKIP_MATCH_LENGTH && level < NO_SKIP_LEVEL ) - { - arrival += match.length - LITERALS_PER_CONTROL; - inputCursor += match.length - LITERALS_PER_CONTROL; - } - } - } - - // If this would cost more to encode than it would if it were just literals, encode it with no control blocks, - // just literals - if ( ( arrivalWatermark->cost + END_PADDING_LITERALS * LITERAL_BITS + CONTROLS_PER_BLOCK * CONTROL_BITS ) > ( inputLength * LITERAL_BITS ) ) - { - memcpy( output, input, inputLength ); - - return inputLength; - } - - Arrival* previousPathNode; - - // now trace the actual optimal parse path back, connecting the nodes in the other direction. - for ( const Arrival* pathNode = arrivalWatermark; pathNode->from > 0; pathNode = previousPathNode ) - { - previousPathNode = state->arrivals + ( pathNode->from - LITERALS_PER_CONTROL ); - - previousPathNode->to = static_cast( ( pathNode - state->arrivals ) + LITERALS_PER_CONTROL ); - } - - uint8_t* outputCursor = output; - - memcpy( outputCursor, input, MIN_MATCH_LENGTH ); - - outputCursor += MIN_MATCH_LENGTH; - - uint8_t* currentControlBlock = outputCursor; - uint32_t currentControlCount = 0; - uint32_t totalControlCount = 0; - - outputCursor += CONTROL_BLOCK_SIZE; - - Arrival* nextPathNode; - - size_t totalPathLength = MIN_MATCH_LENGTH; - uint16_t previousOffset = INITIAL_OFFSET; - - bool lastControlIsNop = false; - - // Now walk forwards again and actually write out the data. - for ( const Arrival* pathNode = state->arrivals; pathNode < arrivalWatermark; pathNode = nextPathNode ) - { - int32_t currentPosition = static_cast< int32_t >( ( pathNode - state->arrivals ) + LITERALS_PER_CONTROL ); - - nextPathNode = state->arrivals + ( pathNode->to - LITERALS_PER_CONTROL ); - - size_t pathDistance = nextPathNode - pathNode; - - totalPathLength += pathDistance; - - lastControlIsNop = false; - - if ( nextPathNode->offset == 0 ) - { - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - currentControlCount = 0; - } - - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - ( static_cast( pathDistance ) - 1 ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - ( static_cast< uint8_t >( pathDistance ) - 1 ) << CONTROL_BITS; - } - - // output the literals. - for ( int32_t where = 0; where < pathDistance; ++where ) - { - const uint8_t* currentInput = input + currentPosition + where; - - outputCursor[ where ] = *currentInput ^ *( currentInput - previousOffset ); - } - - outputCursor += pathDistance; - - ++totalControlCount; - ++currentControlCount; - } - else - { - // Check if we need to start a new control block - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - } - - // Write the offset out - note the xor with the previous offset. - *reinterpret_cast< uint16_t* >( outputCursor ) = nextPathNode->offset ^ previousOffset; - - previousOffset = nextPathNode->offset; - outputCursor += sizeof( uint16_t ); - - if ( pathDistance < EXTENDED_MATCH_BOUND ) - { - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - static_cast( pathDistance ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - static_cast< uint8_t >( pathDistance ) << CONTROL_BITS; - } - - ++currentControlCount; - } - else - { - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - static_cast( EXTENDED_MATCH_BOUND ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - static_cast< uint8_t >( EXTENDED_MATCH_BOUND ) << CONTROL_BITS; - } - - ++currentControlCount; - - size_t toEncode = pathDistance - EXTENDED_MATCH_BOUND; - - for ( ;; ) - { - // Check if we need to start a new control block - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - } - - // If the encode size is greater than we can hold in a control, write out a full match length - // control, subtract full control value from the amount to encode and loop around again. - if ( toEncode >= EXTENDED_MATCH_BOUND ) - { - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - static_cast( EXTENDED_MATCH_BOUND ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - static_cast< uint8_t >( EXTENDED_MATCH_BOUND ) << CONTROL_BITS; - } - - toEncode -= EXTENDED_MATCH_BOUND; - - ++currentControlCount; - } - else // Write out the remaining match length control. Could potentially be zero. - { - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - static_cast( toEncode ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - static_cast< uint8_t >( toEncode ) << CONTROL_BITS; - } - - if ( toEncode == 0 && currentControlCount == 0 ) - { - lastControlIsNop = true; - } - - ++currentControlCount; - - break; - } - } - } - } - } - - if ( lastControlIsNop ) - { - outputCursor -= CONTROL_BLOCK_SIZE; - } - - size_t remainingLiterals = ( input + inputLength ) - inputCursor; - - // copy remaining literals - memcpy( outputCursor, inputCursor, remainingLiterals ); - - outputCursor += remainingLiterals; - - return outputCursor - output; -} - - -size_t LZSSE4_Decompress( const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength ) -{ - const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); - uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); - - // Data was not compressible, just copy initial values - if ( outputLength == inputLength ) - { - memcpy( output, input, outputLength ); - - return inputLength; - } - - const uint8_t* inputCursor = input; - uint8_t* outputCursor = output; - - // The offset starts off as the minimum match length. We actually need it least four - // characters back because we need them to be set to xor out the literals from the match data. - size_t offset = INITIAL_OFFSET; - __m128i previousCarryHi = _mm_setzero_si128(); - - // Copy the initial literals to the output. - for ( uint32_t where = 0; where < MIN_MATCH_LENGTH; ++where ) - { - *( outputCursor++ ) = *( inputCursor++ ); - } - - // Let me be clear, I am usually anti-macro, but they work for this particular (very unusual) case. - // DECODE_STEP is a regular decoding step, DECODE_STEP_HALF and DECODE_STEP_END are because the compiler couldn't - // seem to remove some of the dead code where values were updated and then never used. - - // What these macros do: - // Decode a single literal run or match run for a single control nibble. - // How they do it: - // - Read the *unaligned* input (in the case of LZSSE-F - twice), it goes into both a regular variable and an SSE register, - // because it could either be literals or an offset (or nothing at all). The low byte of streamBytesRead controls how much we advance - // the input cursor. - // - Used a contived set of casts to sign extend the "read offset" control mask and then use it to mask the input word, - // which is then xor'd against the offset, for a "branchless" conditional move into the offset which - // has been carried over from the previous literal/match block. Note, this ends up doing better than a cmov on most - // modern processors. But we need to pre-xor the input offset. - // - We then load the match data from output buffer (offset back from the current output point). Unconditional load here. - // - We broadcast the "from literal" control mask from the current least significant byte of the SSE register using a shuffle epi-8 - // - We mask the literals with that SSE register wide mask. - // - The literals have been pre-xor'd with the data read in as match data, so we use an xor to branchlessly choose between the two. - // In this case, it ends up a better option than a blendv on most processors. - // - Store the block. We store all 16 bytes of the SSE register (due to some constraints in the format of the data, we won't - // go past the end of the buffer), but we may overlap this. - // - bytesOut controls how much we advance the output cursor. - // - We use 8 bit shifts to advance all the controls up to the next byte. There is some variable sized register trickery that - // x86/x64 is great for as long as we don't anger the register renamer. - -#define DECODE_STEP( HILO, CHECKMATCH, CHECKBUFFERS ) \ - { \ - size_t inputWord = *reinterpret_cast( inputCursor ); \ - __m128i literals = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); \ - \ - offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ - \ - readOffsetHalf##HILO >>= 8; \ - \ - const uint8_t* matchPointer = reinterpret_cast( outputCursor - offset ); \ - \ - if ( CHECKMATCH && matchPointer < output ) \ - goto MATCH_UNDERFLOW; \ - \ - __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ - __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ - \ - literals = _mm_and_si128( literals, fromLiteral ); \ - \ - fromLiteral##HILO = _mm_srli_si128( fromLiteral##HILO, 1 ); \ - \ - __m128i toStore = _mm_xor_si128( matchData, literals ); \ - \ - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ - \ - outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ - inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ - \ - bytesOutHalf##HILO >>= 8; \ - streamBytesReadHalf##HILO >>= 8; \ - \ - if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ - goto BUFFER_END; \ - } - -#define DECODE_STEP_HALF( HILO, CHECKMATCH, CHECKBUFFERS ) \ - { \ - size_t inputWord = *reinterpret_cast( inputCursor ); \ - __m128i literals = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); \ - \ - offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ - \ - const uint8_t* matchPointer = reinterpret_cast( outputCursor - offset ); \ - \ - if ( CHECKMATCH && matchPointer < output ) \ - goto MATCH_UNDERFLOW; \ - \ - __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ - __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ - \ - literals = _mm_and_si128( literals, fromLiteral ); \ - \ - fromLiteral##HILO = _mm_srli_si128( fromLiteral##HILO, 1 ); \ - \ - __m128i toStore = _mm_xor_si128( matchData, literals ); \ - \ - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ - \ - outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ - inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ - \ - if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ - goto BUFFER_END; \ - } - -#define DECODE_STEP_END( HILO, CHECKMATCH, CHECKBUFFERS ) \ - { \ - size_t inputWord = *reinterpret_cast( inputCursor ); \ - __m128i literals = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); \ - \ - offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ - \ - const uint8_t* matchPointer = reinterpret_cast( outputCursor - offset ); \ - \ - if ( CHECKMATCH && matchPointer < output ) \ - goto MATCH_UNDERFLOW; \ - \ - __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ - __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ - \ - literals = _mm_and_si128( literals, fromLiteral ); \ - \ - __m128i toStore = _mm_xor_si128( matchData, literals ); \ - \ - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ - \ - outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ - inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ - \ - if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ - goto BUFFER_END; \ - } - -#define DECODE_STEP_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP( Lo, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP( Hi, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_HALF_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_HALF( Lo, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_HALF_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_HALF( Hi, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_END_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_END( Lo, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_END_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_END( Hi, CHECKMATCH, CHECKBUFFERS ) - - __m128i nibbleMask = _mm_set1_epi8( 0xF ); - __m128i offsetSize = _mm_set1_epi8( OFFSET_SIZE ); - - // Note, we use this block here because it allows the "fake" inputEarlyEnd/outputEarlyEnd not to cause register spills - // in the decompression loops. And yes, that did actually happen. - { - -#pragma warning ( push ) -#pragma warning ( disable : 4101 ) - - // These variables are not actually ever used in this block, because we use - // a constant conditional expression to take out the branches that would hit them. - // But unfortunately, we need them to compile. - const uint8_t* inputEarlyEnd; - uint8_t* outputEarlyEnd; - -#pragma warning ( pop ) - - // "Safe" ends to the buffer, before the input/output cursors hit these, we can loop without overflow checks. - const uint8_t* inputSafeEnd = ( input + inputLength ) - INPUT_BUFFER_SAFE; - uint8_t* outputSafeEnd = ( output + outputLength ) - OUTPUT_BUFFER_SAFE; - - // Decoding loop with offset output buffer underflow test, but no buffer overflow tests, assumed to end at a safe distance - // from overflows - while ( ( outputCursor - output ) < LZ_WINDOW_SIZE && outputCursor < outputSafeEnd && inputCursor < inputSafeEnd ) - { - // load the control block - __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); - - // split the control block into high and low nibbles - __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); - __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); - - // Note, the carries are set when the nibble is at its highest value, 15, meaning the operation after will - // be an extension of the current match operation. - - // Work out the carry for the low nibbles (which will be used with the high controls to put them into - // match without offset read mode). - __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); - - // The carry for the high nibbles is used with the low controls, but needs one byte from the previous iteration. We save - // the calculated carry to use that byte next iteration. - __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); - __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); - - previousCarryHi = carryHi; - - // We make the implicit assumption that the maximum number of literals to controls here is twice the offset size (4 vs 2), - // we are doing this here to save keeping the value around (spilling or fetching it each time) - __m128i literalsPerControl = _mm_add_epi8( offsetSize, offsetSize ); - - // Here we are testing if the runs will be literals or matches. Note that if the carries are set from the previous operation - // this will essentially be ignored later on. - __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); - __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); - - // Here we're calculating the number of bytes that will be output, we are actually subtracting negative one from the control - // (handy trick where comparison result masks are negative one) if carry is not set and it is a literal. - __m128i bytesOutLo = _mm_sub_epi8( controlLo, _mm_andnot_si128( shiftedCarryHi, isLiteralLo ) ); - __m128i bytesOutHi = _mm_sub_epi8( controlHi, _mm_andnot_si128( carryLo, isLiteralHi ) ); - - // Calculate the number of bytes to read per control. - // In the case the carry is set, no bytes. Otherwise, the offset size (2 bytes) for matches or the number of output bytes for literals. - __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_blendv_epi8( offsetSize, bytesOutLo, isLiteralLo ) ); - __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_blendv_epi8( offsetSize, bytesOutHi, isLiteralHi ) ); - - // I want 128 set bits please. - __m128i allSet = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); - - // Masks to read the offset (or keep the previous one) - set in the case that this is not a literal and the carry is not set - __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), allSet ); - __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), allSet ); - - // Masks whether we are reading literals - set if the carry is not set and these are literals. - __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); - __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); - - // Advance the input past the control block - inputCursor += CONTROL_BLOCK_SIZE; - - { - // Pull out the bottom halves off the SSE registers from before - we want these - // things in GPRs for the more linear logic. - uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_HALF_LO( true, false ); - DECODE_STEP_HALF_HI( true, false ); - } - - { - // Now the top halves. - uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_END_LO( true, false ); - DECODE_STEP_END_HI( true, false ); - } - } - - // Decoding loop with no buffer checks, but will end at a safe distance from the end of the buffers. - // Note, when we get here we have already reached the point in the output buffer which is *past* where we can underflow - // due to a bad match offset. - while ( outputCursor < outputSafeEnd && inputCursor < inputSafeEnd ) - { - // This code is the same as the loop above, see comments there - __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); - __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); - __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); - - __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); - __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); - __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); // where we take the carry from the previous hi values - - previousCarryHi = carryHi; - - __m128i literalsPerControl = _mm_add_epi8( offsetSize, offsetSize ); - __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); - __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); - - __m128i bytesOutLo = _mm_sub_epi8( controlLo, _mm_andnot_si128( shiftedCarryHi, isLiteralLo ) ); - __m128i bytesOutHi = _mm_sub_epi8( controlHi, _mm_andnot_si128( carryLo, isLiteralHi ) ); - - __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_blendv_epi8( offsetSize, bytesOutLo, isLiteralLo ) ); - __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_blendv_epi8( offsetSize, bytesOutHi, isLiteralHi ) ); - - __m128i neg1 = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); - - __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), neg1 ); - __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), neg1 ); - - __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); - __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); - - inputCursor += CONTROL_BLOCK_SIZE; - - { - uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_HALF_LO( false, false ); - DECODE_STEP_HALF_HI( false, false ); - } - - { - uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_END_LO( false, false ); - DECODE_STEP_END_HI( false, false ); - } - } - } - - // Decoding loop with all buffer checks. - { - const uint8_t* inputEarlyEnd; - uint8_t* outputEarlyEnd; - inputEarlyEnd = ( input + inputLength ) - END_PADDING_LITERALS; - outputEarlyEnd = ( output + outputLength ) - END_PADDING_LITERALS; - - while ( outputCursor < outputEarlyEnd && inputCursor < inputEarlyEnd ) - { - __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); - __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); - __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); - - __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); - __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); - __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); - - previousCarryHi = carryHi; - - __m128i literalsPerControl = _mm_add_epi8( offsetSize, offsetSize ); - __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); - __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); - - __m128i bytesOutLo = _mm_sub_epi8( controlLo, _mm_andnot_si128( shiftedCarryHi, isLiteralLo ) ); - __m128i bytesOutHi = _mm_sub_epi8( controlHi, _mm_andnot_si128( carryLo, isLiteralHi ) ); - - __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_blendv_epi8( offsetSize, bytesOutLo, isLiteralLo ) ); - __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_blendv_epi8( offsetSize, bytesOutHi, isLiteralHi ) ); - - __m128i neg1 = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); - - __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), neg1 ); - __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), neg1 ); - - __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); - __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); - - inputCursor += CONTROL_BLOCK_SIZE; - - if ( inputCursor > inputEarlyEnd ) - goto BUFFER_END; - - { - uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_HALF_LO( true, true ); - DECODE_STEP_HALF_HI( true, true ); - } - - { - // Now the top halves. - uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_END_LO( true, true ); - DECODE_STEP_END_HI( true, true ); - } - } - -BUFFER_END: - - // When we get here, we have either advanced the right amount on both cursors - // or something bad happened, so leave it as is, so we can tell where - // the error happened. - if ( inputCursor == inputEarlyEnd && outputCursor == outputEarlyEnd ) - { - size_t remainingLiterals = ( input + inputLength ) - inputCursor; - - // copy any trailing literals - memcpy( outputCursor, inputCursor, remainingLiterals ); - - outputCursor += remainingLiterals; - } - } - -MATCH_UNDERFLOW: - - return outputCursor - output; -} diff --git a/contrib/lzsse/lzsse4/lzsse4.h b/contrib/lzsse/lzsse4/lzsse4.h deleted file mode 100644 index 0fa5e6c8ed0..00000000000 --- a/contrib/lzsse/lzsse4/lzsse4.h +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright (c) 2016, Conor Stokes -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef LZSSE4_H__ -#define LZSSE4_H__ - -#pragma once - -/* LZSSE4 - x64/SSE targeted codec for better performance with lower compression ratio data/less optimal compressors. - * Supports minimum 4 byte matches, maximum 15 bytes of match per control word and 4 byte literal runs per control word. - */ - -#ifdef __cplusplus -extern "C" -{ -#endif - -/* Re-usable parse state object for compression. */ -typedef struct LZSSE4_FastParseState LZSSE4_FastParseState; - -/* Allocate the parse state for compression - returns null on failure */ -LZSSE4_FastParseState* LZSSE4_MakeFastParseState(); - -/* De-allocate the parse state for compression */ -void LZSSE4_FreeFastParseState( LZSSE4_FastParseState* toFree ); - -/* Re-usable parse state object for compression. */ -typedef struct LZSSE4_OptimalParseState LZSSE4_OptimalParseState; - -/* Allocate the parse state for compression - returns null on failure. Note - Buffersize has to be greater or equal to any inputLength used with LZSSE2_CompressOptimalParse */ -LZSSE4_OptimalParseState* LZSSE4_MakeOptimalParseState( size_t bufferSize ); - -/* De-allocate the parse state for compression */ -void LZSSE4_FreeOptimalParseState( LZSSE4_OptimalParseState* toFree ); - - -/* "Optimal" compression routine. -* Will compress data into LZSSE4 format, uses hash BST matching to find matches and run an optimal parse (high relative memory usage). Requires SSE 4.1. -* state : Contains the hash table for matching, passed as a parameter so that allocations can be re-used. -* input : Buffer containing uncompressed data to be compressed. May not be null. -* inputLength : Length of the compressed data in the input buffer - note should be under 2GB. -* output : Buffer that will receive the compressed output. -* outputLength : The length reserved in the buffer for compressed data. This should be at least inputLength. Note, -* The compressed data should never be longer than inputLength, as in this case the data is stored raw. -* level : The compression level to use for this file 1->17, 17 delivers the highest compression, 1 delivers the least. -* Thread Safety - state can not be used on multiple threads with calls running concurrently. Can run multiple threads with separate state -* concurrently. -* -* Returns the size of the compressed data, or 0 in the case of error (e.g. outputLength is less than inputLength). -*/ -size_t LZSSE4_CompressOptimalParse( LZSSE4_OptimalParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength, unsigned int level ); - - -/* "Fast" compression routine. - * Will compress data into LZSSE4 format, uses a simple single entry hash/greedy matching to find matches. Requires SSE 4.1. - * state : Contains the hash table for matching, passed as a parameter so that allocations can be re-used. - * input : Buffer containing uncompressed data to be compressed. May not be null. - * inputLength : Length of the compressed data in the input buffer - note should be under 2GB. - * output : Buffer that will receive the compressed output. - * outputLength : The length reserved in the buffer for compressed data. This should be at least inputLength. Note, - * The compressed data should never be longer than inputLength, as in this case the data is stored raw. - * - * Thread Safety - state can not be used on multiple threads with calls running concurrently. Can run multiple threads with separate state - * concurrently. - * - * Returns the size of the compressed data, or 0 in the case of error (e.g. outputLength is less than inputLength). - */ -size_t LZSSE4_CompressFast( LZSSE4_FastParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength ); - -/* Decompression routine. - * This routine will decompress data in the LZSSE4 format and currently requires SSE 4.1 and is targeted at x64. - * It will perform poorly on x86 due to hunger for registers. - * input : Buffer containing compressed input block. May not be null. - * inputLength : Length of the compressed data in the input buffer - note, this should be under 2GB - * output : Buffer that will received the de-compressed output. Note, that this needs to be at least outputLength long. - * May not be null. - * outputLength : The length of the compressed output - note, this should be under 2GB - * - * Provided that input and output are valid pointers to buffers of at least their specified size, this routine - * should be memory safe - both match pointer checks and input/output buffer checks exist. - * - * Returns the size of the decompressed data, which will be less than outputLength in the event of an error (number of bytes - * will indicate where in the output stream the error occured). - * - * Note that this data is not hash verified, errors that occur are either from a misformed stream or bad buffer sizes. - * Remember, corrupt data can still be valid to decompress. - */ -size_t LZSSE4_Decompress( const void* input, size_t inputLength, void* output, size_t outputLength ); - -#ifdef __cplusplus -} -#endif - -#endif /* -- LZSSE4_H__ */ diff --git a/contrib/lzsse/lzsse4/lzsse4_platform.h b/contrib/lzsse/lzsse4/lzsse4_platform.h deleted file mode 100644 index d1d3f1cced2..00000000000 --- a/contrib/lzsse/lzsse4/lzsse4_platform.h +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright (c) 2016, Brian Marshall -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef LZSSE4_PLATFORM_H__ -#define LZSSE4_PLATFORM_H__ - -#pragma once - -/* - Compiler/Platform detection based on the table from: - https://blogs.msdn.microsoft.com/vcblog/2015/12/04/clang-with-microsoft-codegen-in-vs-2015-update-1/ -*/ - -#ifdef _MSC_VER - -/* - Microsoft Visual Studio Support. - C1xx/C2, Clang/C2 and Clang/LLVM all support the Microsoft header files and _BitScanForward - - Note: if you receive errors with the intrinsics make sure that you have SSE4.1 support enabled. - For example with Clang include "-msse4.1" on the command line -*/ -#include - -#else /* _MSC_VER */ - -#ifdef __GNUC__ - -/* - GCC -*/ - -/* - Note: including just would be sufficient, but including x86intrin is a better match to intrin.h on Visual Studio as - both include all intrinsics for the enabled processor, rather than just SSE4.1. -*/ -#include -/* _BitScanForward is Visual Studio specific. */ -#define _BitScanForward(x, m) *(x) = __builtin_ctz(m) - -#else - -/* -If you hit the error below, then add detection for your compiler/platform to this header file. -*/ -#error Platform not supported - -#endif /* __GNUC__ */ -#endif /* _MSC_VER */ - -#endif /* -- LZSSE4_PLATFORM_H__ */ diff --git a/contrib/lzsse/lzsse8/lzsse8.cpp b/contrib/lzsse/lzsse8/lzsse8.cpp deleted file mode 100644 index a5682a1de81..00000000000 --- a/contrib/lzsse/lzsse8/lzsse8.cpp +++ /dev/null @@ -1,1568 +0,0 @@ -/* -Copyright (c) 2016, Conor Stokes -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#include -#include -#include -#include - -#include "lzsse8_platform.h" -#include "lzsse8.h" - -#pragma warning ( disable : 4127 ) - -namespace -{ - // Constants - most of these should not be changed without corresponding code changes because it will break many things in unpredictable ways. - const uint32_t WINDOW_BITS = 16; - const uint32_t MIN_MATCH_LENGTH = 4; - const uint32_t LZ_WINDOW_SIZE = 1 << WINDOW_BITS; - const uint32_t LZ_WINDOW_MASK = LZ_WINDOW_SIZE - 1; - const uint32_t FAST_HASH_BITS = 20; // You can change this - more bits = more matches, less bits = more cache hits - const uint32_t FAST_BUCKETS_COUNT = 1 << FAST_HASH_BITS; - const uint32_t FAST_HASH_MASK = FAST_BUCKETS_COUNT - 1; - const uint32_t MIN_COMPRESSION_SIZE = 32; - const uint32_t END_PADDING_LITERALS = 16; - const int32_t NO_MATCH = -1; - const int32_t EMPTY_NODE = -1; - const uint32_t CONTROL_BITS = 4; - const uint32_t OFFSET_SIZE = 2; - const uint32_t EXTENDED_MATCH_BOUND = ( 1 << CONTROL_BITS ) - 1; - const uint32_t CONTROL_BLOCK_SIZE = sizeof( __m128i ); - const uint32_t CONTROLS_PER_BLOCK = 32; - const uint32_t LITERALS_PER_CONTROL = 8; - const uint32_t MAX_INPUT_PER_CONTROL = 8; - const uint32_t INITIAL_MATCH_BOUND = 11; - const size_t OUTPUT_BUFFER_SAFE = EXTENDED_MATCH_BOUND * CONTROLS_PER_BLOCK; - const size_t INPUT_BUFFER_SAFE = MAX_INPUT_PER_CONTROL * CONTROLS_PER_BLOCK; - const uint16_t INITIAL_OFFSET = LITERALS_PER_CONTROL; - const uint32_t OFFSET_BITS = 16; - const uint32_t BASE_MATCH_BITS = OFFSET_BITS + CONTROL_BITS; - const uint32_t OPTIMAL_HASH_BITS = 20; - const uint32_t OPTIMAL_BUCKETS_COUNT = 1 << OPTIMAL_HASH_BITS; - const uint32_t OPTIMAL_HASH_MASK = OPTIMAL_BUCKETS_COUNT - 1; - const uint32_t LITERAL_BITS = 8; - const size_t SKIP_MATCH_LENGTH = 128; - const uint32_t NO_SKIP_LEVEL = 17; - -} - - -struct LZSSE8_FastParseState -{ - int32_t buckets[ FAST_BUCKETS_COUNT ]; // stores the first matching position, we can then look at the rest of the matches by tracing through the window. -}; - - -LZSSE8_FastParseState* LZSSE8_MakeFastParseState() -{ - return new LZSSE8_FastParseState(); -} - - -void LZSSE8_FreeFastParseState( LZSSE8_FastParseState* toFree ) -{ - delete toFree; -} - - -inline void SetHash( LZSSE8_FastParseState* state, uint32_t hash, const uint8_t* input, const uint8_t* inputCursor ) -{ - int32_t position = static_cast( inputCursor - input ); - - state->buckets[ hash & FAST_HASH_MASK ] = position; -} - -// Simple fast hash function - actually what is used in snappy and derivatives -// There's probably better, but I haven't spent time focusing on this area yet. -inline uint32_t HashFast( const uint8_t* inputCursor ) -{ - return *reinterpret_cast( inputCursor ) * 0x1e35a7bd >> ( 32 - FAST_HASH_BITS ); -} - -struct Arrival -{ - size_t cost; - int32_t from; - int32_t to; - uint16_t offset; -}; - -struct TreeNode -{ - int32_t children[ 2 ]; -}; - -struct LZSSE8_OptimalParseState -{ - int32_t roots[ OPTIMAL_BUCKETS_COUNT ]; - - TreeNode window[ LZ_WINDOW_SIZE ]; - - Arrival* arrivals; - - size_t bufferSize; -}; - - -LZSSE8_OptimalParseState* LZSSE8_MakeOptimalParseState( size_t bufferSize ) -{ - if ( bufferSize > 0 && ( SIZE_MAX / sizeof( Arrival ) ) < bufferSize ) - { - return nullptr; - } - - LZSSE8_OptimalParseState* result = reinterpret_cast< LZSSE8_OptimalParseState* >( ::malloc( sizeof( LZSSE8_OptimalParseState ) ) ); - - result->bufferSize = bufferSize; - - if ( result != nullptr ) - { - result->arrivals = reinterpret_cast< Arrival* >( ::malloc( sizeof( Arrival ) * bufferSize ) ); - - if ( result->arrivals == nullptr ) - { - LZSSE8_FreeOptimalParseState( result ); - - result = nullptr; - } - } - - return result; -} - - -void LZSSE8_FreeOptimalParseState( LZSSE8_OptimalParseState* toFree ) -{ - ::free( toFree->arrivals ); - - toFree->arrivals = nullptr; - - ::free( toFree ); -} - - -inline uint32_t HashOptimal( const uint8_t* inputCursor ) -{ - return *reinterpret_cast( inputCursor ) * 0x1e35a7bd >> ( 32 - OPTIMAL_HASH_BITS ); -} - - -struct Match -{ - size_t length; - int32_t position; - uint16_t offset; -}; - - -inline Match SearchAndUpdateFinder( LZSSE8_OptimalParseState& state, const uint8_t* input, const uint8_t* inputCursor, const uint8_t* inputEnd, uint32_t cutOff ) -{ - Match result; - - int32_t position = static_cast( inputCursor - input ); - - result.position = NO_MATCH; - result.length = MIN_MATCH_LENGTH; - result.offset = 0; - - size_t lengthToEnd = inputEnd - inputCursor; - int32_t lastPosition = position - ( LZ_WINDOW_SIZE - 1 ); - uint32_t hash = HashOptimal( inputCursor ); - - lastPosition = lastPosition > 0 ? lastPosition : 0; - - int32_t treeCursor = state.roots[ hash ]; - - state.roots[ hash ] = position; - - int32_t* left = &state.window[ position & LZ_WINDOW_MASK ].children[ 1 ]; - int32_t* right = &state.window[ position & LZ_WINDOW_MASK ].children[ 0 ]; - size_t leftLength = 0; - size_t rightLength = 0; - - for ( ;; ) - { - if ( cutOff-- == 0 || treeCursor < lastPosition ) - { - *left = *right = EMPTY_NODE; - break; - } - - TreeNode& currentNode = state.window[ treeCursor & LZ_WINDOW_MASK ]; - const uint8_t* key = input + treeCursor; - size_t matchLength = leftLength < rightLength ? leftLength : rightLength; - - uint16_t matchOffset = static_cast( position - treeCursor ); - size_t maxLength = matchOffset <= ( EXTENDED_MATCH_BOUND + 1 ) && matchOffset < lengthToEnd ? matchOffset : lengthToEnd; - - while ( matchLength < lengthToEnd ) - { - __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); - __m128i match16 = _mm_loadu_si128( reinterpret_cast( key + matchLength ) ); - - unsigned long matchBytes; - - _BitScanForward( &matchBytes, ( static_cast( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); - - matchLength += matchBytes; - - if ( matchBytes != 16 ) - { - break; - } - } - - matchLength = matchLength < lengthToEnd ? matchLength : lengthToEnd; - - size_t truncatedMatchLength = matchLength < maxLength ? matchLength : maxLength; - - if ( truncatedMatchLength >= result.length && matchOffset >= LITERALS_PER_CONTROL ) - { - result.length = truncatedMatchLength; - result.offset = matchOffset; - result.position = treeCursor; - } - - if ( matchLength == lengthToEnd ) - { - *left = currentNode.children[ 1 ]; - *right = currentNode.children[ 0 ]; - break; - } - - if ( inputCursor[ matchLength ] < key[ matchLength ] || ( matchLength == lengthToEnd ) ) - { - *left = treeCursor; - left = currentNode.children; - treeCursor = *left; - leftLength = matchLength; - } - else - { - *right = treeCursor; - right = currentNode.children + 1; - treeCursor = *right; - rightLength = matchLength; - } - } - - // Special RLE overlapping match case, the LzFind style match above doesn't work very well with our - // restriction of overlapping matches having offsets of at least 16. - // Suffix array seems like a better option to handling this. - { - // Note, we're detecting long RLE here, but if we have an offset too close, we'll sacrifice a fair - // amount of decompression performance to load-hit-stores. - int32_t matchPosition = position - ( sizeof( __m128i ) * 2 ); - - if ( matchPosition >= 0 ) - { - uint16_t matchOffset = static_cast( position - matchPosition ); - const uint8_t* key = input + matchPosition; - size_t matchLength = 0; - - while ( matchLength < lengthToEnd ) - { - __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); - __m128i match16 = _mm_loadu_si128( reinterpret_cast( key + matchLength ) ); - - unsigned long matchBytes; - - _BitScanForward( &matchBytes, ( static_cast( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); - - matchLength += matchBytes; - - if ( matchBytes != 16 ) - { - break; - } - - } - - matchLength = matchLength < lengthToEnd ? matchLength : lengthToEnd; - - if ( matchLength >= result.length ) - { - result.length = matchLength; - result.offset = matchOffset; - result.position = matchPosition; - } - } - } - - return result; -} - - -size_t LZSSE8_CompressOptimalParse( LZSSE8_OptimalParseState* state, const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength, unsigned int level ) -{ - if ( outputLength < inputLength || state->bufferSize < inputLength ) - { - // error case, output buffer not large enough. - return 0; - } - - const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); - uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); - - if ( inputLength < MIN_COMPRESSION_SIZE ) - { - memcpy( output, input, inputLength ); - - return inputLength; - } - - const uint8_t* inputCursor = input; - const uint8_t* inputEnd = input + inputLength; - Arrival* arrivalWatermark = state->arrivals; - Arrival* arrival = state->arrivals; - uint32_t cutOff = 1 << level; - - for ( int32_t* rootCursor = state->roots, *end = rootCursor + OPTIMAL_BUCKETS_COUNT; rootCursor < end; rootCursor += 4 ) - { - rootCursor[ 0 ] = EMPTY_NODE; - rootCursor[ 1 ] = EMPTY_NODE; - rootCursor[ 2 ] = EMPTY_NODE; - rootCursor[ 3 ] = EMPTY_NODE; - } - - for ( uint32_t where = 0; where < LITERALS_PER_CONTROL; ++where ) - { - SearchAndUpdateFinder( *state, input, inputCursor, inputEnd - END_PADDING_LITERALS, cutOff ); - - ++inputCursor; - } - - arrival->cost = LITERAL_BITS * LITERALS_PER_CONTROL; - arrival->from = -1; - arrival->offset = 0; - - // loop through each character and project forward the matches at that character to calculate the cheapest - // path of arrival for each individual character. - for ( const uint8_t* earlyEnd = inputEnd - END_PADDING_LITERALS; inputCursor < earlyEnd; ++inputCursor, ++arrival ) - { - uint32_t lengthToEnd = static_cast< uint32_t >( earlyEnd - inputCursor ); - int32_t currentPosition = static_cast< int32_t >( inputCursor - input ); - size_t literalsForward = LITERALS_PER_CONTROL < lengthToEnd ? LITERALS_PER_CONTROL : lengthToEnd; - size_t arrivalCost = arrival->cost; - - // NOTE - we currently assume only 2 literals filled in here, because the minimum match length is 3. - // If we wanted to go with a higher minimum match length, we would need to fill in more literals before hand. - // Also, because there is a maximum of 2 literals per control block assumed. - - // project forward the cost of a single literal - - for ( size_t where = 1; where <= literalsForward; ++where ) - { - Arrival* literalArrival = arrival + where; - size_t literalCost = arrivalCost + CONTROL_BITS + ( where * LITERAL_BITS ); - - if ( literalArrival > arrivalWatermark || literalArrival->cost > literalCost ) - { - literalArrival->cost = literalCost; - literalArrival->from = currentPosition; - literalArrival->offset = 0; - - arrivalWatermark = literalArrival > arrivalWatermark ? literalArrival : arrivalWatermark; - } - } - - Match match = SearchAndUpdateFinder( *state, input, inputCursor, earlyEnd, cutOff ); - - if ( match.position != NO_MATCH ) - { - for ( size_t matchedLength = MIN_MATCH_LENGTH, end = match.length + 1; matchedLength < end; ++matchedLength ) - { - Arrival* matchArrival = arrival + matchedLength; - size_t matchCost = arrivalCost + BASE_MATCH_BITS; - - if ( matchedLength >= INITIAL_MATCH_BOUND ) - { - matchCost += ( ( ( matchedLength - INITIAL_MATCH_BOUND ) / EXTENDED_MATCH_BOUND ) + 1 ) * CONTROL_BITS; - } - - if ( matchArrival > arrivalWatermark || matchArrival->cost > matchCost ) - { - matchArrival->cost = matchCost; - matchArrival->from = currentPosition; - matchArrival->offset = match.offset; - - arrivalWatermark = matchArrival > arrivalWatermark ? matchArrival : arrivalWatermark; - } - } - - if ( match.length > SKIP_MATCH_LENGTH && level < NO_SKIP_LEVEL ) - { - arrival += match.length - LITERALS_PER_CONTROL; - inputCursor += match.length - LITERALS_PER_CONTROL; - } - } - } - - // If this would cost more to encode than it would if it were just literals, encode it with no control blocks, - // just literals - if ( ( arrivalWatermark->cost + END_PADDING_LITERALS * LITERAL_BITS + CONTROLS_PER_BLOCK * CONTROL_BITS ) > ( inputLength * LITERAL_BITS ) ) - { - memcpy( output, input, inputLength ); - - return inputLength; - } - - Arrival* previousPathNode; - - // now trace the actual optimal parse path back, connecting the nodes in the other direction. - for ( const Arrival* pathNode = arrivalWatermark; pathNode->from > 0; pathNode = previousPathNode ) - { - previousPathNode = state->arrivals + ( pathNode->from - LITERALS_PER_CONTROL ); - - previousPathNode->to = static_cast( ( pathNode - state->arrivals ) + LITERALS_PER_CONTROL ); - } - - uint8_t* outputCursor = output; - - memcpy( outputCursor, input, LITERALS_PER_CONTROL ); - - outputCursor += LITERALS_PER_CONTROL; - - uint8_t* currentControlBlock = outputCursor; - uint32_t currentControlCount = 0; - uint32_t totalControlCount = 0; - - outputCursor += CONTROL_BLOCK_SIZE; - - Arrival* nextPathNode; - - size_t totalPathLength = LITERALS_PER_CONTROL; - uint16_t previousOffset = INITIAL_OFFSET; - - bool lastControlIsNop = false; - - // Now walk forwards again and actually write out the data. - for ( const Arrival* pathNode = state->arrivals; pathNode < arrivalWatermark; pathNode = nextPathNode ) - { - int32_t currentPosition = static_cast< int32_t >( ( pathNode - state->arrivals ) + LITERALS_PER_CONTROL ); - - nextPathNode = state->arrivals + ( pathNode->to - LITERALS_PER_CONTROL ); - - size_t pathDistance = nextPathNode - pathNode; - - totalPathLength += pathDistance; - - lastControlIsNop = false; - - if ( nextPathNode->offset == 0 ) - { - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - currentControlCount = 0; - } - - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - ( static_cast( pathDistance ) - 1 ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - ( static_cast< uint8_t >( pathDistance ) - 1 ) << CONTROL_BITS; - } - - // output the literals. - for ( int32_t where = 0; where < pathDistance; ++where ) - { - const uint8_t* currentInput = input + currentPosition + where; - - outputCursor[ where ] = *currentInput ^ *( currentInput - previousOffset ); - } - - outputCursor += pathDistance; - - ++totalControlCount; - ++currentControlCount; - } - else - { - // Check if we need to start a new control block - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - } - - // Write the offset out - note the xor with the previous offset. - *reinterpret_cast< uint16_t* >( outputCursor ) = nextPathNode->offset ^ previousOffset; - - previousOffset = nextPathNode->offset; - outputCursor += sizeof( uint16_t ); - - if ( pathDistance < INITIAL_MATCH_BOUND ) - { - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - static_cast( pathDistance + MIN_MATCH_LENGTH ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - static_cast< uint8_t >( pathDistance + MIN_MATCH_LENGTH ) << CONTROL_BITS; - } - - ++currentControlCount; - } - else - { - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - static_cast( EXTENDED_MATCH_BOUND ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - static_cast< uint8_t >( EXTENDED_MATCH_BOUND ) << CONTROL_BITS; - } - - ++currentControlCount; - - size_t toEncode = pathDistance - INITIAL_MATCH_BOUND; - - for ( ;; ) - { - // Check if we need to start a new control block - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - } - - // If the encode size is greater than we can hold in a control, write out a full match length - // control, subtract full control value from the amount to encode and loop around again. - if ( toEncode >= EXTENDED_MATCH_BOUND ) - { - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - static_cast( EXTENDED_MATCH_BOUND ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - static_cast< uint8_t >( EXTENDED_MATCH_BOUND ) << CONTROL_BITS; - } - - toEncode -= EXTENDED_MATCH_BOUND; - - ++currentControlCount; - } - else // Write out the remaining match length control. Could potentially be zero. - { - if ( ( currentControlCount & 1 ) == 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] = - static_cast( toEncode ); - } - else - { - currentControlBlock[ currentControlCount >> 1 ] |= - static_cast< uint8_t >( toEncode ) << CONTROL_BITS; - } - - if ( toEncode == 0 && currentControlCount == 0 ) - { - lastControlIsNop = true; - } - - ++currentControlCount; - - break; - } - } - } - } - } - - if ( lastControlIsNop ) - { - outputCursor -= CONTROL_BLOCK_SIZE; - } - - size_t remainingLiterals = ( input + inputLength ) - inputCursor; - - // copy remaining literals - memcpy( outputCursor, inputCursor, remainingLiterals ); - - outputCursor += remainingLiterals; - - return outputCursor - output; -} - - -size_t LZSSE8_CompressFast( LZSSE8_FastParseState* state, const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength ) -{ - if ( outputLength < inputLength ) - { - // error case, output buffer not large enough. - return 0; - } - - const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); - uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); - - if ( inputLength < MIN_COMPRESSION_SIZE ) - { - memcpy( output, input, inputLength ); - - return inputLength; - } - - const uint8_t* inputCursor = input; - const uint8_t* inputEnd = input + inputLength; - const uint8_t* inputEarlyEnd = inputEnd - END_PADDING_LITERALS; - uint8_t* outputCursor = output; - uint8_t* outputEarlyEnd = ( output + outputLength ) - END_PADDING_LITERALS; - uint32_t hash = 0; - - // initialize hash to empty - for ( int32_t* where = state->buckets, *end = state->buckets + FAST_BUCKETS_COUNT; where < end; where += 4 ) - { - where[ 0 ] = -1; - where[ 1 ] = -1; - where[ 2 ] = -1; - where[ 3 ] = -1; - } - - // initial literals that wont be compressed - for ( uint32_t where = 0; where < LITERALS_PER_CONTROL; ++where ) - { - hash = HashFast( inputCursor ); - - SetHash( state, hash, input, inputCursor ); - - *( outputCursor++ ) = *( inputCursor++ ); - } - - uint8_t* currentControlBlock = outputCursor; - uint32_t currentControlCount = 0; - uint16_t previousOffset = INITIAL_OFFSET; - size_t literalsToFlush = 0; - - outputCursor += CONTROL_BLOCK_SIZE; - - bool lastControlIsNop = false; - - // Loop through the data until we hit the end of one of the buffers (minus the end padding literals) - while ( inputCursor < inputEarlyEnd && outputCursor <= outputEarlyEnd ) - { - lastControlIsNop = false; - - hash = HashFast( inputCursor ); - - int matchPosition = state->buckets[ hash & FAST_HASH_MASK ]; - int currentPosition = static_cast< int32_t >( inputCursor - input ); - uint32_t matchLength = 0; - uint16_t matchOffset = static_cast< uint16_t >( currentPosition - matchPosition ); - - // If we had a hit in the hash and it wasn't outside the window. - if ( matchPosition >= 0 && ( currentPosition - matchPosition ) < ( LZ_WINDOW_SIZE - 1 ) && matchOffset >= LITERALS_PER_CONTROL ) - { - const uint8_t* matchCandidate = input + matchPosition; - uint32_t lengthToEnd = static_cast< uint32_t >( inputEarlyEnd - inputCursor ); - // Here we limit the hash length to prevent overlap matches with offset less than 16 bytes - uint32_t maxLength = matchOffset <= ( EXTENDED_MATCH_BOUND + 1 ) && matchOffset < lengthToEnd ? matchOffset : lengthToEnd; - - // Find how long the match is 16 bytes at a time. - while ( matchLength < maxLength ) - { - __m128i input16 = _mm_loadu_si128( reinterpret_cast( inputCursor + matchLength ) ); - __m128i match16 = _mm_loadu_si128( reinterpret_cast( matchCandidate + matchLength ) ); - - unsigned long matchBytes; - - // Finds the number of equal bytes at the start of the 16 - _BitScanForward( &matchBytes, ( static_cast< unsigned long >( ~_mm_movemask_epi8( _mm_cmpeq_epi8( input16, match16 ) ) ) | 0x10000 ) ); - - matchLength += matchBytes; - - if ( matchBytes != sizeof( __m128i ) ) - { - break; - } - } - - matchLength = matchLength < maxLength ? matchLength : maxLength; - } - - // If we have at least the minimum match length (4 bytes) - if ( matchLength >= MIN_MATCH_LENGTH ) - { - // Do we have literals to flush before the match? - if ( literalsToFlush > 0 ) - { - // Start a new control block if we need one. - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - - // Would be larger than compressed size, get out! - if ( outputCursor > outputEarlyEnd ) - { - break; - } - } - - currentControlBlock[ currentControlCount >> 1 ] = - ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( literalsToFlush - 1 ) << 4 ); - - // flush the literals. - // note the xor against the data that would be read in the match. - for ( uint32_t where = 0; where < literalsToFlush; ++where ) - { - const uint8_t* currentInput = inputCursor - ( literalsToFlush - where ); - - *( outputCursor++ ) = *currentInput ^ *( currentInput - previousOffset ); - } - - ++currentControlCount; - - literalsToFlush = 0; - - // Would be larger than compressed size, get out! - if ( outputCursor > outputEarlyEnd ) - { - break; - } - } - - // Start a new control block if the previous one is full. - // Note this is done before the offset is written out - the offset - // is always written after the control block containing the first - // control in the match. - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - - if ( outputCursor > outputEarlyEnd ) - { - break; - } - } - - // The match length value we are encoding. - - // Write the offset out - note the xor with the previous offset. - *reinterpret_cast< uint16_t* >( outputCursor ) = matchOffset ^ previousOffset; - - previousOffset = matchOffset; - outputCursor += sizeof( uint16_t ); - - if ( matchLength < INITIAL_MATCH_BOUND ) - { - currentControlBlock[ currentControlCount >> 1 ] = - ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( matchLength + MIN_MATCH_LENGTH ) << 4 ); - - ++currentControlCount; - } - else - { - currentControlBlock[ currentControlCount >> 1 ] = - ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( EXTENDED_MATCH_BOUND ) << 4 ); - - ++currentControlCount; - - size_t toEncode = matchLength - INITIAL_MATCH_BOUND; - - for ( ;; ) - { - // Check if we need to start a new control block - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - - if ( outputCursor > outputEarlyEnd ) - { - break; - } - } - - // If the encode size is greater than we can hold in a control, write out a full match length - // control, subtract full control value from the amount to encode and loop around again. - if ( toEncode >= EXTENDED_MATCH_BOUND ) - { - currentControlBlock[ currentControlCount >> 1 ] = - ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( EXTENDED_MATCH_BOUND ) << 4 ); - - toEncode -= EXTENDED_MATCH_BOUND; - - ++currentControlCount; - } - else // Write out the remaining match length control. Could potentially be zero. - { - currentControlBlock[ currentControlCount >> 1 ] = - ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( toEncode ) << 4 ); - - if ( currentControlBlock == 0 && toEncode == 0 ) - { - lastControlIsNop = true; - } - - ++currentControlCount; - - break; - } - } - } - - // Update the value into the hash for future matches. - SetHash( state, hash, input, inputCursor ); - - ++inputCursor; - - // Hash all the other values in the match too. - for ( const uint8_t* nextArrival = inputCursor + matchLength - 1; inputCursor < nextArrival; ++inputCursor ) - { - hash = HashFast( inputCursor ); - SetHash( state, hash, input, inputCursor ); - } - } - else - { - // One more literal to write out. - ++literalsToFlush; - - // If we have reached the maximum number of literals allowed in the control, flush them out. - if ( literalsToFlush == LITERALS_PER_CONTROL ) - { - // Check if the control block is full and we need start a new one. - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - - if ( outputCursor > outputEarlyEnd ) - { - break; - } - } - - currentControlBlock[ currentControlCount >> 1 ] = - ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( ( static_cast( LITERALS_PER_CONTROL - 1 ) ) << 4 ); - - ++currentControlCount; - - *reinterpret_cast< uint64_t* >( outputCursor ) = - *reinterpret_cast< const uint64_t* >( inputCursor - ( sizeof( uint64_t ) - 1 ) ) ^ - *reinterpret_cast< const uint64_t* >( ( inputCursor - ( sizeof( uint64_t ) - 1 ) ) - previousOffset ); - - outputCursor += sizeof( uint64_t ); - - //*( outputCursor++ ) = *( inputCursor - 3 ) ^ *( ( inputCursor - 3 ) - previousOffset ); - //*( outputCursor++ ) = *( inputCursor - 2 ) ^ *( ( inputCursor - 2 ) - previousOffset ); - //*( outputCursor++ ) = *( inputCursor - 1 ) ^ *( ( inputCursor - 1 ) - previousOffset ); - //*( outputCursor++ ) = *inputCursor ^ *( inputCursor - previousOffset ); - - if ( outputCursor > outputEarlyEnd ) - { - break; - } - - literalsToFlush = 0; - } - - // Update the hash with this byte - SetHash( state, hash, input, inputCursor ); - - ++inputCursor; - } - } - - // If we would create a compression output bigger than or equal to the input, just copy the input to the output and return equal size. - if ( ( ( outputCursor + literalsToFlush + ( currentControlCount == CONTROLS_PER_BLOCK ? CONTROL_BLOCK_SIZE : 0 ) ) ) >= output + inputLength - END_PADDING_LITERALS ) - { - memcpy( output, input, inputLength ); - - outputCursor = output + inputLength; - } - else - { - // Flush any remaining literals. - if ( literalsToFlush > 0 ) - { - lastControlIsNop = false; - - if ( currentControlCount == CONTROLS_PER_BLOCK ) - { - currentControlBlock = outputCursor; - outputCursor += CONTROL_BLOCK_SIZE; - - _mm_storeu_si128( reinterpret_cast< __m128i* >( outputCursor ), _mm_setzero_si128() ); - - currentControlCount = 0; - } - - currentControlBlock[ currentControlCount >> 1 ] = - ( currentControlBlock[ currentControlCount >> 1 ] >> 4 ) | ( static_cast( literalsToFlush - 1 ) << 4 ); - - for ( uint32_t where = 0; where < literalsToFlush; ++where ) - { - const uint8_t* currentInput = inputCursor - ( literalsToFlush - where ); - - *( outputCursor++ ) = *currentInput ^ *( currentInput - previousOffset ); - } - - ++currentControlCount; - } - - // Need to finish off shifting the final control block into the low nibble if there is no second nibble - if ( ( currentControlCount & 1 ) > 0 ) - { - currentControlBlock[ currentControlCount >> 1 ] >>= 4; - } - - if ( lastControlIsNop ) - { - outputCursor -= CONTROL_BLOCK_SIZE; - } - - size_t remainingLiterals = ( input + inputLength ) - inputCursor; - - // copy remaining literals - memcpy( outputCursor, inputCursor, remainingLiterals ); - - outputCursor += remainingLiterals; - } - - // Return the size of the compressed data. - return outputCursor - output; -} - - -size_t LZSSE8_Decompress( const void* inputChar, size_t inputLength, void* outputChar, size_t outputLength ) -{ - const uint8_t* input = reinterpret_cast< const uint8_t* >( inputChar ); - uint8_t* output = reinterpret_cast< uint8_t* >( outputChar ); - - // Data was not compressible, just copy initial values - if ( outputLength == inputLength ) - { - memcpy( output, input, outputLength ); - - return inputLength; - } - - const uint8_t* inputCursor = input; - uint8_t* outputCursor = output; - - // The offset starts off as the minimum match length. We actually need it least four - // characters back because we need them to be set to xor out the literals from the match data. - size_t offset = INITIAL_OFFSET; - __m128i previousCarryHi = _mm_setzero_si128(); - - // Copy the initial literals to the output. - for ( uint32_t where = 0; where < LITERALS_PER_CONTROL; ++where ) - { - *( outputCursor++ ) = *( inputCursor++ ); - } - - // Let me be clear, I am usually anti-macro, but they work for this particular (very unusual) case. - // DECODE_STEP is a regular decoding step, DECODE_STEP_HALF and DECODE_STEP_END are because the compiler couldn't - // seem to remove some of the dead code where values were updated and then never used. - - // What these macros do: - // Decode a single literal run or match run for a single control nibble. - // How they do it: - // - Read the *unaligned* input (in the case of LZSSE-F - twice), it goes into both a regular variable and an SSE register, - // because it could either be literals or an offset (or nothing at all). The low byte of streamBytesRead controls how much we advance - // the input cursor. - // - Used a contived set of casts to sign extend the "read offset" control mask and then use it to mask the input word, - // which is then xor'd against the offset, for a "branchless" conditional move into the offset which - // has been carried over from the previous literal/match block. Note, this ends up doing better than a cmov on most - // modern processors. But we need to pre-xor the input offset. - // - We then load the match data from output buffer (offset back from the current output point). Unconditional load here. - // - We broadcast the "from literal" control mask from the current least significant byte of the SSE register using a shuffle epi-8 - // - We mask the literals with that SSE register wide mask. - // - The literals have been pre-xor'd with the data read in as match data, so we use an xor to branchlessly choose between the two. - // In this case, it ends up a better option than a blendv on most processors. - // - Store the block. We store all 16 bytes of the SSE register (due to some constraints in the format of the data, we won't - // go past the end of the buffer), but we may overlap this. - // - bytesOut controls how much we advance the output cursor. - // - We use 8 bit shifts to advance all the controls up to the next byte. There is some variable sized register trickery that - // x86/x64 is great for as long as we don't anger the register renamer. - -#define DECODE_STEP( HILO, CHECKMATCH, CHECKBUFFERS ) \ - { \ - size_t inputWord = *reinterpret_cast( inputCursor ); \ - __m128i literals = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); \ - \ - offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ - \ - readOffsetHalf##HILO >>= 8; \ - \ - const uint8_t* matchPointer = reinterpret_cast( outputCursor - offset ); \ - \ - if ( CHECKMATCH && matchPointer < output ) \ - goto MATCH_UNDERFLOW; \ - \ - __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ - __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ - \ - literals = _mm_and_si128( literals, fromLiteral ); \ - \ - fromLiteral##HILO = _mm_srli_si128( fromLiteral##HILO, 1 ); \ - \ - __m128i toStore = _mm_xor_si128( matchData, literals ); \ - \ - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ - \ - outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ - inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ - \ - bytesOutHalf##HILO >>= 8; \ - streamBytesReadHalf##HILO >>= 8; \ - \ - if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ - goto BUFFER_END; \ - } - -#define DECODE_STEP_HALF( HILO, CHECKMATCH, CHECKBUFFERS ) \ - { \ - size_t inputWord = *reinterpret_cast( inputCursor ); \ - __m128i literals = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); \ - \ - offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ - \ - const uint8_t* matchPointer = reinterpret_cast( outputCursor - offset ); \ - \ - if ( CHECKMATCH && matchPointer < output ) \ - goto MATCH_UNDERFLOW; \ - \ - __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ - __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ - \ - literals = _mm_and_si128( literals, fromLiteral ); \ - \ - fromLiteral##HILO = _mm_srli_si128( fromLiteral##HILO, 1 ); \ - \ - __m128i toStore = _mm_xor_si128( matchData, literals ); \ - \ - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ - \ - outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ - inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ - \ - if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ - goto BUFFER_END; \ - } - -#define DECODE_STEP_END( HILO, CHECKMATCH, CHECKBUFFERS ) \ - { \ - size_t inputWord = *reinterpret_cast( inputCursor ); \ - __m128i literals = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); \ - \ - offset ^= static_cast( static_cast( static_cast( readOffsetHalf##HILO ) ) ) & inputWord; \ - \ - const uint8_t* matchPointer = reinterpret_cast( outputCursor - offset ); \ - \ - if ( CHECKMATCH && matchPointer < output ) \ - goto MATCH_UNDERFLOW; \ - \ - __m128i fromLiteral = _mm_shuffle_epi8( fromLiteral##HILO, _mm_setzero_si128() ); \ - __m128i matchData = _mm_loadu_si128( reinterpret_cast( matchPointer ) ); \ - \ - literals = _mm_and_si128( literals, fromLiteral ); \ - \ - __m128i toStore = _mm_xor_si128( matchData, literals ); \ - \ - _mm_storeu_si128( reinterpret_cast<__m128i*>( outputCursor ), toStore ); \ - \ - outputCursor += static_cast< uint8_t >( bytesOutHalf##HILO ); \ - inputCursor += static_cast< uint8_t >( streamBytesReadHalf##HILO ); \ - \ - if ( CHECKBUFFERS && ( outputCursor >= outputEarlyEnd || inputCursor > inputEarlyEnd ) ) \ - goto BUFFER_END; \ - } - -#define DECODE_STEP_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP( Lo, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP( Hi, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_HALF_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_HALF( Lo, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_HALF_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_HALF( Hi, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_END_LO(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_END( Lo, CHECKMATCH, CHECKBUFFERS ) -#define DECODE_STEP_END_HI(CHECKMATCH, CHECKBUFFERS ) DECODE_STEP_END( Hi, CHECKMATCH, CHECKBUFFERS ) - - __m128i nibbleMask = _mm_set1_epi8( 0xF ); - __m128i literalsPerControl = _mm_set1_epi8( static_cast< char >( LITERALS_PER_CONTROL ) ); - __m128i bytesInOutLUT = _mm_set_epi8( '\x2B', '\x2A', '\x29', '\x28', '\x27', '\x26', '\x25', '\x24', '\x88', '\x77', '\x66', '\x55', '\x44', '\x33', '\x22', '\x11' ); - - // Note, we use this block here because it allows the "fake" inputEarlyEnd/outputEarlyEnd not to cause register spills - // in the decompression loops. And yes, that did actually happen. - { - -#pragma warning ( push ) -#pragma warning ( disable : 4101 ) - - // These variables are not actually ever used in this block, because we use - // a constant conditional expression to take out the branches that would hit them. - // But unfortunately, we need them to compile. - const uint8_t* inputEarlyEnd; - uint8_t* outputEarlyEnd; - -#pragma warning ( pop ) - - // "Safe" ends to the buffer, before the input/output cursors hit these, we can loop without overflow checks. - const uint8_t* inputSafeEnd = ( input + inputLength ) - INPUT_BUFFER_SAFE; - uint8_t* outputSafeEnd = ( output + outputLength ) - OUTPUT_BUFFER_SAFE; - - // Decoding loop with offset output buffer underflow test, but no buffer overflow tests, assumed to end at a safe distance - // from overflows - while ( ( outputCursor - output ) < LZ_WINDOW_SIZE && outputCursor < outputSafeEnd && inputCursor < inputSafeEnd ) - { - // load the control block - __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); - - // split the control block into high and low nibbles - __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); - __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); - - // Note, the carries are set when the nibble is at its highest value, 15, meaning the operation after will - // be an extension of the current match operation. - - // Work out the carry for the low nibbles (which will be used with the high controls to put them into - // match without offset read mode). - __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); - - // The carry for the high nibbles is used with the low controls, but needs one byte from the previous iteration. We save - // the calculated carry to use that byte next iteration. - __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); - __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); - - previousCarryHi = carryHi; - - // We make the implicit assumption that the maximum number of literals to controls here is twice the offset size (4 vs 2), - // we are doing this here to save keeping the value around (spilling or fetching it each time) - - __m128i streamBytesLo = _mm_shuffle_epi8( bytesInOutLUT, controlLo ); - __m128i streamBytesHi = _mm_shuffle_epi8( bytesInOutLUT, controlHi ); - - // Here we're calculating the number of bytes that will be output, we are actually subtracting negative one from the control - // (handy trick where comparison result masks are negative one) if carry is not set and it is a literal. - __m128i bytesOutLo = _mm_blendv_epi8( _mm_and_si128( streamBytesLo, nibbleMask ), controlLo, shiftedCarryHi ); - __m128i bytesOutHi = _mm_blendv_epi8( _mm_and_si128( streamBytesHi, nibbleMask ), controlHi, carryLo ); - - // Calculate the number of bytes to read per control. - // In the case the carry is set, no bytes. Otherwise, the offset size (2 bytes) for matches or the number of output bytes for literals. - __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_and_si128( _mm_srli_epi32( streamBytesLo, 4 ), nibbleMask ) ); - __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_and_si128( _mm_srli_epi32( streamBytesHi, 4 ), nibbleMask ) ); - - // Here we are testing if the runs will be literals or matches. Note that if the carries are set from the previous operation - // this will essentially be ignored later on. - __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); - __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); - - // I want 128 set bits please. - __m128i allSet = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); - - // Masks to read the offset (or keep the previous one) - set in the case that this is not a literal and the carry is not set - __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), allSet ); - __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), allSet ); - - // Masks whether we are reading literals - set if the carry is not set and these are literals. - __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); - __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); - - // Advance the input past the control block - inputCursor += CONTROL_BLOCK_SIZE; - - { - // Pull out the bottom halves off the SSE registers from before - we want these - // things in GPRs for the more linear logic. - uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_HALF_LO( true, false ); - DECODE_STEP_HALF_HI( true, false ); - } - - { - // Now the top halves. - uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - - DECODE_STEP_LO( true, false ); - DECODE_STEP_HI( true, false ); - DECODE_STEP_END_LO( true, false ); - DECODE_STEP_END_HI( true, false ); - } - } - - // Decoding loop with no buffer checks, but will end at a safe distance from the end of the buffers. - // Note, when we get here we have already reached the point in the output buffer which is *past* where we can underflow - // due to a bad match offset. - while ( outputCursor < outputSafeEnd && inputCursor < inputSafeEnd ) - { - // This code is the same as the loop above, see comments there - - // load the control block - __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); - - // split the control block into high and low nibbles - __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); - __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); - - // Note, the carries are set when the nibble is at its highest value, 15, meaning the operation after will - // be an extension of the current match operation. - - // Work out the carry for the low nibbles (which will be used with the high controls to put them into - // match without offset read mode). - __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); - - // The carry for the high nibbles is used with the low controls, but needs one byte from the previous iteration. We save - // the calculated carry to use that byte next iteration. - __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); - __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); - - previousCarryHi = carryHi; - - // We make the implicit assumption that the maximum number of literals to controls here is twice the offset size (4 vs 2), - // we are doing this here to save keeping the value around (spilling or fetching it each time) - - __m128i streamBytesLo = _mm_shuffle_epi8( bytesInOutLUT, controlLo ); - __m128i streamBytesHi = _mm_shuffle_epi8( bytesInOutLUT, controlHi ); - - // Here we're calculating the number of bytes that will be output, we are actually subtracting negative one from the control - // (handy trick where comparison result masks are negative one) if carry is not set and it is a literal. - __m128i bytesOutLo = _mm_blendv_epi8( _mm_and_si128( streamBytesLo, nibbleMask ), controlLo, shiftedCarryHi ); - __m128i bytesOutHi = _mm_blendv_epi8( _mm_and_si128( streamBytesHi, nibbleMask ), controlHi, carryLo ); - - // Calculate the number of bytes to read per control. - // In the case the carry is set, no bytes. Otherwise, the offset size (2 bytes) for matches or the number of output bytes for literals. - __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_and_si128( _mm_srli_epi32( streamBytesLo, 4 ), nibbleMask ) ); - __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_and_si128( _mm_srli_epi32( streamBytesHi, 4 ), nibbleMask ) ); - - // Here we are testing if the runs will be literals or matches. Note that if the carries are set from the previous operation - // this will essentially be ignored later on. - __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); - __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); - - // I want 128 set bits please. - __m128i allSet = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); - - // Masks to read the offset (or keep the previous one) - set in the case that this is not a literal and the carry is not set - __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), allSet ); - __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), allSet ); - - // Masks whether we are reading literals - set if the carry is not set and these are literals. - __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); - __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); - - inputCursor += CONTROL_BLOCK_SIZE; - - { - uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_HALF_LO( false, false ); - DECODE_STEP_HALF_HI( false, false ); - } - - { - uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - - DECODE_STEP_LO( false, false ); - DECODE_STEP_HI( false, false ); - DECODE_STEP_END_LO( false, false ); - DECODE_STEP_END_HI( false, false ); - } - } - } - - // Decoding loop with all buffer checks. - { - const uint8_t* inputEarlyEnd; - uint8_t* outputEarlyEnd; - inputEarlyEnd = ( input + inputLength ) - END_PADDING_LITERALS; - outputEarlyEnd = ( output + outputLength ) - END_PADDING_LITERALS; - - while ( outputCursor < outputEarlyEnd && inputCursor < inputEarlyEnd ) - { - // This code is the same as the loop above, see comments there - - // load the control block - __m128i controlBlock = _mm_loadu_si128( reinterpret_cast( inputCursor ) ); - - // split the control block into high and low nibbles - __m128i controlHi = _mm_and_si128( _mm_srli_epi32( controlBlock, CONTROL_BITS ), nibbleMask ); - __m128i controlLo = _mm_and_si128( controlBlock, nibbleMask ); - - // Note, the carries are set when the nibble is at its highest value, 15, meaning the operation after will - // be an extension of the current match operation. - - // Work out the carry for the low nibbles (which will be used with the high controls to put them into - // match without offset read mode). - __m128i carryLo = _mm_cmpeq_epi8( controlLo, nibbleMask ); - - // The carry for the high nibbles is used with the low controls, but needs one byte from the previous iteration. We save - // the calculated carry to use that byte next iteration. - __m128i carryHi = _mm_cmpeq_epi8( controlHi, nibbleMask ); - __m128i shiftedCarryHi = _mm_alignr_epi8( carryHi, previousCarryHi, 15 ); - - previousCarryHi = carryHi; - - // Note, unlike the other compressors, we are essentially doing an in register lookup table to implement the logic here. - __m128i streamBytesLo = _mm_shuffle_epi8( bytesInOutLUT, controlLo ); - __m128i streamBytesHi = _mm_shuffle_epi8( bytesInOutLUT, controlHi ); - - // Either use the value from the lookup, or in the case the carry is set, use the control value. - __m128i bytesOutLo = _mm_blendv_epi8( _mm_and_si128( streamBytesLo, nibbleMask ), controlLo, shiftedCarryHi ); - __m128i bytesOutHi = _mm_blendv_epi8( _mm_and_si128( streamBytesHi, nibbleMask ), controlHi, carryLo ); - - // Calculate the number of bytes to read per control. - // We use the value from the lookup table to . - __m128i streamBytesReadLo = _mm_andnot_si128( shiftedCarryHi, _mm_and_si128( _mm_srli_epi32( streamBytesLo, 4 ), nibbleMask ) ); - __m128i streamBytesReadHi = _mm_andnot_si128( carryLo, _mm_and_si128( _mm_srli_epi32( streamBytesHi, 4 ), nibbleMask ) ); - - // Here we are testing if the runs will be literals or matches. Note that if the carries are set from the previous operation - // this will essentially be ignored later on. - __m128i isLiteralHi = _mm_cmplt_epi8( controlHi, literalsPerControl ); - __m128i isLiteralLo = _mm_cmplt_epi8( controlLo, literalsPerControl ); - - // I want 128 set bits please. - __m128i allSet = _mm_cmpeq_epi8( shiftedCarryHi, shiftedCarryHi ); - - // Masks to read the offset (or keep the previous one) - set in the case that this is not a literal and the carry is not set - __m128i readOffsetLo = _mm_xor_si128( _mm_or_si128( isLiteralLo, shiftedCarryHi ), allSet ); - __m128i readOffsetHi = _mm_xor_si128( _mm_or_si128( isLiteralHi, carryLo ), allSet ); - - // Masks whether we are reading literals - set if the carry is not set and these are literals. - __m128i fromLiteralLo = _mm_andnot_si128( shiftedCarryHi, isLiteralLo ); - __m128i fromLiteralHi = _mm_andnot_si128( carryLo, isLiteralHi ); - - inputCursor += CONTROL_BLOCK_SIZE; - - if ( inputCursor > inputEarlyEnd ) - goto BUFFER_END; - - { - uint64_t bytesOutHalfLo = static_cast( _mm_cvtsi128_si64( bytesOutLo ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_cvtsi128_si64( bytesOutHi ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_cvtsi128_si64( streamBytesReadLo ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_cvtsi128_si64( streamBytesReadHi ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_cvtsi128_si64( readOffsetLo ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_cvtsi128_si64( readOffsetHi ) ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_HALF_LO( true, true ); - DECODE_STEP_HALF_HI( true, true ); - } - - { - // Now the top halves. - uint64_t bytesOutHalfLo = static_cast( _mm_extract_epi64( bytesOutLo, 1 ) ); - uint64_t bytesOutHalfHi = static_cast( _mm_extract_epi64( bytesOutHi, 1 ) ); - - uint64_t streamBytesReadHalfLo = static_cast( _mm_extract_epi64( streamBytesReadLo, 1 ) ); - uint64_t streamBytesReadHalfHi = static_cast( _mm_extract_epi64( streamBytesReadHi, 1 ) ); - - uint64_t readOffsetHalfLo = static_cast( _mm_extract_epi64( readOffsetLo, 1 ) ); - uint64_t readOffsetHalfHi = static_cast( _mm_extract_epi64( readOffsetHi, 1 ) ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - - DECODE_STEP_LO( true, true ); - DECODE_STEP_HI( true, true ); - DECODE_STEP_END_LO( true, true ); - DECODE_STEP_END_HI( true, true ); - } - } - -BUFFER_END: - - // When we get here, we have either advanced the right amount on both cursors - // or something bad happened, so leave it as is, so we can tell where - // the error happened. - if ( inputCursor == inputEarlyEnd && outputCursor == outputEarlyEnd ) - { - size_t remainingLiterals = ( input + inputLength ) - inputCursor; - - // copy any trailing literals - memcpy( outputCursor, inputCursor, remainingLiterals ); - - outputCursor += remainingLiterals; - } - } - -MATCH_UNDERFLOW: - - return outputCursor - output; -} diff --git a/contrib/lzsse/lzsse8/lzsse8.h b/contrib/lzsse/lzsse8/lzsse8.h deleted file mode 100644 index dec5297a17c..00000000000 --- a/contrib/lzsse/lzsse8/lzsse8.h +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright (c) 2016, Conor Stokes -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef LZSSE8_H__ -#define LZSSE8_H__ - -#pragma once - -/* LZSSE8 - x64/SSE targeted codec for better performance with lower compression ratio data/less optimal compressors. -* Supports minimum 4 byte matches, maximum 15 bytes of match per control word and 8 byte literal runs per control word. -*/ - -#ifdef __cplusplus -extern "C" -{ -#endif - -/* Re-usable parse state object for compression. */ -typedef struct LZSSE8_FastParseState LZSSE8_FastParseState; - -/* Allocate the parse state for compression - returns null on failure */ -LZSSE8_FastParseState* LZSSE8_MakeFastParseState(); - -/* De-allocate the parse state for compression */ -void LZSSE8_FreeFastParseState( LZSSE8_FastParseState* toFree ); - -/* Re-usable parse state object for compression. */ -typedef struct LZSSE8_OptimalParseState LZSSE8_OptimalParseState; - -/* Allocate the parse state for compression - returns null on failure. Note -Buffersize has to be greater or equal to any inputLength used with LZSSE2_CompressOptimalParse */ -LZSSE8_OptimalParseState* LZSSE8_MakeOptimalParseState( size_t bufferSize ); - -/* De-allocate the parse state for compression */ -void LZSSE8_FreeOptimalParseState( LZSSE8_OptimalParseState* toFree ); - - -/* "Optimal" compression routine. -* Will compress data into LZSSE8 format, uses hash BST matching to find matches and run an optimal parse (high relative memory usage). Requires SSE 4.1. -* state : Contains the hash table for matching, passed as a parameter so that allocations can be re-used. -* input : Buffer containing uncompressed data to be compressed. May not be null. -* inputLength : Length of the compressed data in the input buffer - note should be under 2GB. -* output : Buffer that will receive the compressed output. -* outputLength : The length reserved in the buffer for compressed data. This should be at least inputLength. Note, -* The compressed data should never be longer than inputLength, as in this case the data is stored raw. -* level : The compression level to use for this file 1->17, 17 delivers the highest compression, 1 delivers the least. -* Thread Safety - state can not be used on multiple threads with calls running concurrently. Can run multiple threads with separate state -* concurrently. -* -* Returns the size of the compressed data, or 0 in the case of error (e.g. outputLength is less than inputLength). -*/ -size_t LZSSE8_CompressOptimalParse( LZSSE8_OptimalParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength, unsigned int level ); - - -/* "Fast" compression routine. -* Will compress data into LZSSE8 format, uses a simple single entry hash/greedy matching to find matches. Requires SSE 4.1. -* state : Contains the hash table for matching, passed as a parameter so that allocations can be re-used. -* input : Buffer containing uncompressed data to be compressed. May not be null. -* inputLength : Length of the compressed data in the input buffer - note should be under 2GB. -* output : Buffer that will receive the compressed output. -* outputLength : The length reserved in the buffer for compressed data. This should be at least inputLength. Note, -* The compressed data should never be longer than inputLength, as in this case the data is stored raw. -* -* Thread Safety - state can not be used on multiple threads with calls running concurrently. Can run multiple threads with separate state -* concurrently. -* -* Returns the size of the compressed data, or 0 in the case of error (e.g. outputLength is less than inputLength). -*/ -size_t LZSSE8_CompressFast( LZSSE8_FastParseState* state, const void* input, size_t inputLength, void* output, size_t outputLength ); - -/* Decompression routine. -* This routine will decompress data in the LZSSE8 format and currently requires SSE 4.1 and is targeted at x64. -* It will perform poorly on x86 due to hunger for registers. -* input : Buffer containing compressed input block. May not be null. -* inputLength : Length of the compressed data in the input buffer - note, this should be under 2GB -* output : Buffer that will received the de-compressed output. Note, that this needs to be at least outputLength long. -* May not be null. -* outputLength : The length of the compressed output - note, this should be under 2GB -* -* Provided that input and output are valid pointers to buffers of at least their specified size, this routine -* should be memory safe - both match pointer checks and input/output buffer checks exist. -* -* Returns the size of the decompressed data, which will be less than outputLength in the event of an error (number of bytes -* will indicate where in the output stream the error occured). -* -* Note that this data is not hash verified, errors that occur are either from a misformed stream or bad buffer sizes. -* Remember, corrupt data can still be valid to decompress. -*/ -size_t LZSSE8_Decompress( const void* input, size_t inputLength, void* output, size_t outputLength ); - -#ifdef __cplusplus -} -#endif - -#endif /* -- LZSSE8_H__ */ diff --git a/contrib/lzsse/lzsse8/lzsse8_platform.h b/contrib/lzsse/lzsse8/lzsse8_platform.h deleted file mode 100644 index 5ad2e0a8853..00000000000 --- a/contrib/lzsse/lzsse8/lzsse8_platform.h +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright (c) 2016, Brian Marshall -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef LZSSE8_PLATFORM_H__ -#define LZSSE8_PLATFORM_H__ - -#pragma once - -/* - Compiler/Platform detection based on the table from: - https://blogs.msdn.microsoft.com/vcblog/2015/12/04/clang-with-microsoft-codegen-in-vs-2015-update-1/ -*/ - -#ifdef _MSC_VER - -/* - Microsoft Visual Studio Support. - C1xx/C2, Clang/C2 and Clang/LLVM all support the Microsoft header files and _BitScanForward - - Note: if you receive errors with the intrinsics make sure that you have SSE4.1 support enabled. - For example with Clang include "-msse4.1" on the command line -*/ -#include - -#else /* _MSC_VER */ - -#ifdef __GNUC__ - -/* - GCC -*/ - -/* - Note: including just would be sufficient, but including x86intrin is a better match to intrin.h on Visual Studio as - both include all intrinsics for the enabled processor, rather than just SSE4.1. -*/ -#include -/* _BitScanForward is Visual Studio specific. */ -#define _BitScanForward(x, m) *(x) = __builtin_ctz(m) - -#else - -/* -If you hit the error below, then add detection for your compiler/platform to this header file. -*/ -#error Platform not supported - -#endif /* __GNUC__ */ -#endif /* _MSC_VER */ - -#endif /* -- LZSSE8_PLATFORM_H__ */ diff --git a/contrib/lzsse/premake4.exe b/contrib/lzsse/premake4.exe deleted file mode 100644 index 50f4c86b528f881a6e56a76228e0fcd32ad512c7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 518144 zcmeFaeSDME`S_nUX&WFk0RvR2TC{4lh}EJ}gP0bQ7LbxsNvKHCsVs|3863ig8J|O{ZAj+r6)dG>UeO(A17@s zxcHBgE{T6_RmF;x*L`*66<@Eo@``J(y)Ietm8&aOrmn5{+O-w4=C@XS{kp5Jo-%s$ z$jV&Pzcv?NWPNb%^8Dw$t$$kn7vA67`oZ#F>+e^V*YSPcxFgFC=yFq+|B!F>U#~2G zTjzhc`~&@N>F)(!yD~0q{huI@gdC2G3yK_5pP73}zRW&HVZo>ZkHfJXOnhH3;c)a^ zT;Oo{c<2=D1nuWShohJ``IrB81l>0BNOL-Z8+AdcCzbX0;FlyiX8pRv5z$X>$3D9F zw|_pjqce}3f6E+7=>Ol6qvr@XAOBD7<*H0cUcENS^Lf`W9)U~7Y7cd=(F%v-@+m8? zx*~an!_i(xri}dpzE=nihW-UX$P~MnWBGZccb7OE?{r`{uA7Egic_5Z>Bnq-Aw;EaPa2Pv0 zW`8U@&fPiF(bb(gLDH`~(cy^2eK*Z?$lIeQNGV^UaFOM;>t&C3INEBv+k#7%Y^S}} zXv|#imPTXhkuw|)o(f}TpZW#Q#*Q*`Z^F?U&5rZfJscNPUfOSsPo@2sx@@TR(bB-Z zgKf7j)~!c@A)E4K$N8vl_8op};-SNDB|V9=1h&bgK+CiybOt~{%15ahMap9kc`dK!cZENf5PWtWSNL2k681#6o!rE?g-azdi+tm+g z#r(SsK`g!*SmSpKLdKN?MkG6~LZC^x%{>V>RgF^}nGV;dzle6dlyqfZvWm0g{1jN4 zhiu~k#oP@+)oLyQs;dAcekU0T;QBosG8w&Hmy>!oi5dMwS7p-GccifQkJ=1M`UgYh%ozh*qQ=Z+K7i6B8&Qn`r#HHplX5XhjS=yc&yR7azl9(DnqdREBT7Edx@914XnY03{tqP9*L5g)oN@JlBNG~c7AiIM{TcEt%BBXa!h~kgYmdtjaC@#j zCt z-Lv0y_(kKLf);7BDc7dYZqqJ!2L+Apsk+mjF;3|;S0HMaSUJ>b%2nHKoo;lv94TKW z?2ZTsHGsf$yk{(P<;3c~vixjl*b$GluO}QcTLp&iodT6K7058ukD&pxpYAgA-7N^n z$+Q%sP`V_&)TRClXxrMBwwv2+IXEb_FQEx7wG)&cH&u6U%k4*~5{q|2NYYgyZ!|tk zZa3xuv*mV6t4$e&i=*nyar740?!84sMQc>u56KEGcsJcbf(UT+MrFR=KumQ3JJMB? zOX;N00Gx7-2G|;1-5cTUL;3>BL^i5^N=E!1po*$S9uoyob%%UAqw4Qtq;V;90y=^D z|1z63cGN(FXD7als;wvkGc;iK2M@p9HZj|Fa)0Wk=uFk=7hv~#Z*7HKpvm^dLiULx zQFU`(dsrU~S9Fu|po6e!!w~jOfQ-fO%OUK=3IB?)@uXtI|%(J7ZDS3*aMkUMC{k+GrHSvQQC5!D|tqHOC4ic>5SbU$(XwVrR znZGDF0(+-|t$5WYLF5rKv#A47)x?9KN@PyFM(6#NBoTi5j2)H31lj|+lIK%WsN5ff zqv7Xv_4}#A*m?TVfg9}fBNOmVfk}N{`^z^9F#ww%AZf< z&&TqoPyT!)e?F8y|ByciBAL*R_%r~HC=34O4NZI+FanjaBU0O~B~957Xts7QCt(Oe z^r$D(_2pCO01p0C>>u7CO9cE!s`}#}-M@M<0FsfEU zS~9d`X-r;!uJs=FCDJ3Nvl0%YzbJKe?dzm0!gh-)CxS;L!joeKiDg!UEe@`uYAj+S zk7+_I2z%d;*vO?t)fY&CnXN+mzd-Rz8#m}SrXxKb4O*ws#py@2vXxc_Yd=A4V@HL# zeQ?yjJbT!vJ251GOTw^GW3olH5N=g3BCLS3u+{8WPwQv1&u$r#X%S!r%ne{Uq8jX8 zc^MqeW-TM80p}z=jjie|k!U#$%M=6`s}pn3KvGn#OpUM`j94!{IYgi|fAM*m6z#q) z;zcVk}xa&E1PyF%eUN7Bs6-#i>Wc4pP%KO)feo1T&FlvuXP>r z-r7ZR<49p@MCTJ<`#;3&Q_n)=&mrf{dG0L*)6Ys)OusByKD{zoHvObjap&~~mC4dg-AQ@6Ac^K) zvK@mYO$uSr<`d*=)9}+eS`XCL^7DXFYFz9Oz4q?Ci*%m2~YaJ}YcDg%qt4 z&TexaHZ^=R13wVRytgvutX&yV)e=*WGTfty?&nj6SimiV{}l;CEP#23F$3}n7=|AD zF1UJdet%nA;5g@Xy9d;Z`PsA|2&BoWopqOaA0Cr9tKB>6&k$Y8b-`bf&dn1V?EwXL zB@4G4Ti;pdNd4Jzch;SiD&3hUl-Yq@YyOcawDJAeEf83%xieywBnpk4g)sQ9G$(nG zIRA)d9}m; z8xJC0eiXEVVCf{9-{Q^9)8>qzDWaVJ*8JN?hRMet&@A@DF!`97Jcz><+jOjbJ!XDU zcf5nOcCu!xd}V8}11oH*PsLgjS$dCaaAd6!Zp^rzkYaUZsaN8QHnZUQaIdJM4?#|A zqJY8yUAUaW$BI?B67BHjlKxM;lSAmPoS{dvQ~?K(col7^22`7E&{%IG4RXN6YO}Gp zdwdYAUq_?y6*r6Bv=gP1J$K(LjXdMXbi_B2tS*3z;G3V3kSGwl^kLk6#)Qc$5s4uZy*!pqD^9wi&WC zK8w;%$?(Nk#{U(+m;~DTt2_&CNdGg>d1yTX+?RmI43E?8094YIa9Dn7%?5%YL|MlQ zRL>qWh@umTH%q@qO~X;+yh5N#0j<@t)N0W8{6Eew-(Ba6gavG55d<@dtI@*?Q80 z@yB^%)IXJ+JvwKC)bfFLl5dyfNbgEtBM82@l zK9kiDYl+lIslJu!mRkhCJwV^qrki>Xrf2Se%L3k`rQYjRgB+_(HaG%k_R)-d@8!D@ zAyl{C)3MSF`ApYVEf7=Apf$Slvoll2OlvAgl^WfJG`tYD4ndwN_&%bpt?kJcAVP=g+UgeO+Aa`Dd>K=PfM~zQ zFQFLzJG6zdLnPM;QFRE+iyeGSG=3SmX)SToU0N?=aLT~M(7xa_dxBH}=73l-s8@9_ z(%S5cs@rZ98t59xW?AHz{n^^6Iv!Bsab!bIZmzen+LcXt#k&wyr%+n!pr{J{nHe*C z;djjz;)OUA&ABulLT&S0NF8&p`YCT>3ipG%I!upqTh)2MFD~3EGLE)D-?~ZFeKF^! zQT0bKb9nE>?w~f94&krY_GVCgJNd56g>K^pcd>19nPWR|D0U<(&3PW&e(Mf+A8sfK z?Aq`yw9@mA%jj`CpGmv73Z|EAhe$m!^9G-Bq+s3gfi;yI%FJ26^Wny!aN2D)mYcJF zGhlT4?9TMC%#bI$!j-KNG4}l>niX>CZnMcBXsSvTnZc$d+s%5H=`e#Hohkiv9qO0} zpj+486n6By=Q8JbjGjK{^MN^TZ$ppOv9%A$9O&y9jcrM`^BZn+rxlNB;T1TXjBPyf z5e1U9fj(%F(FOh0N6ethbep|B`&>r1+xu`!iE~%rbKQZrxyL%g4El`j%i)rF?o9n6 zbGF}{?J`H0vps2dZ_v#elC?jL8Qo1eNe0SP{Cjf^>t+taqmy+(y{#v!xvs zC}u`{X2YWNoJIE7?Rv=f%yi7VX4r2wy8}(`mCkgNJJ;Ae z&!a(>GmuKQG4RZW2I-y3VNXxL%h?mywxOnLcgGYnTwWb^x0xhW#$M>8*BTEQzx3izRtD9j?{SZwQ7~Q!q=jDmq}&+ z(3(OhaPfS4kcEaUN}acm7WoF^oE1*a@Ja7hG(V2LF>9! z)pjkFtz~A%te%fXTJ3yqd_Rld!fG|=`ON67o_9tDo>?=(+*aKqHJs^=U|lr025$Y{ zb?l>cjB~UlN1EYaAUtceWu3qjTF06l5#gP1#40sAF42#dSWe>wJu3)?n@yh;)quz9{Za57QJ=%V=DtrcXU~GFq0$m|FcZ z=vw1Whx#(OkfO2PP`}Kv`fQ@mMwLBXIfwTkl*ijpM%?Bz#{QzDZ}#k@#|-<-TW-IT zx>gYYHM!ZB5KrzgfIrB5$T7Gon4{JFpr8K&@!RZ1HngB$n z9pw9-T(>+73wpevlphScSx^_Sn#4)l!444PDEbJZW~aY<6lv&lU}Q;ZU_m<6 zU)@-NN47r_f9}*EPRbv}G{nY{Y&*JQzTb?w11+K}THH%>^N;qd?XC-Xyx-|I=h*nf z=yDhhZeL#BQOHh85_Rof8Z!zq=H0?u~vU>Z!&FfbM zj!ejBPionZWvHG(o)*`+NBrh6H*Fz-9c&*ujQZTT7&o##KV&S)VMG2moip$?x)RdQ z>6{-!esuPq6jn~>AWnz6n&!Y)JzQ;In&pg5l{cAlaRjocj3sKE{vT+Pr zZKVY44Q`Dip?!9{=lEp8>RMKK@38gOAPv8Kh-r$tA<6$BsdB_C2EyPz(%$6cmv1Y^A=hAfBjjBM{E%)57$i z7N-3}yror_+wL@5qvr1)R_EX3uU!pO8DOUmsc~3~fpAr7bbJvaII6;pG9W$4zdQ(r_O!K0u`|V})JAX^o!^(PNAucf zS)519fE+-bN0Sh-bphfAeLb9-)Kb(9(N!%uV6tq!8gkS~}qzkbHSR%xG5 zi1OuZ7(}gDG-{bYZ^|%ndAS;LFEg8_B4d`No2CwlQjP8zbbufS#J1Yqbxr+ACkQfc zVA)})qU2OI8O7R3-?CL(JG0C}XtQf~GN5sIu34zDccx%(@s%2TRz<$d1YKrKuFMFE z98VGGmeI1f`&Y~8!B3dGs5>*KuO%~QpPr9nWH4ov-tRdE&?$WW+GNW#$c_sMyN)MbvjbruV(#9C;mA zc_Z_&kWe2uwG^u=tXE3db`2~5-0jBoZr8dK%n^Egk?b;(yW8kVxnK7@bo z!@*(=0yr0;wTs@6gS%+SksOT)!~!Q(V&JhxMh!3cLtXG_8w>3%s{~!lI8dHzGL zZLD{(vF3w5ZDWl^=8MV+9egu%@bx7FVl3t&k!7v>g^xMYZ|BlVs9In?s zvys_Vz96dWZQcRhgKftkM`;z_ayU5xiSLJK@F5E+9hH~c) zYtSb)z23?(iP|WGXlQPzXoW6%p%|B`B4byU{Gd%saFfxWO_gk2fdC`f)pg)jd7!TB zA&a03e43nr{`Otd+gBj=N5@&U-C}lNSV!H?J(+qR@m4SkD;$?z;ka4h=moovV)Ijf z&Wqlb7QIPSdnjDD#?^5-jE|=-0_mD>sga_=QdshdBUD;M2nBrhPc5R&`7RhcaoskGX;duKocbcOI_<;#IO4sMp=8PEnNL+!5UZF;z7ZojgOLQB!MS+ zTr`qdaMm8n{z8o!hf_+gY|TC)_3w(eOT0Ut_=>NK zx$Rbe$Z1a=@5PQJk||NV$R-oI^cWX85R)OFahjvoAr|1)8-Z$}L-zx6Oc|L7T*X|DNn%I5v&Kz`NU`1SAFUc~E*_!7|k1;NH5k+Bh z5eBX>rbpEwKWUn0zr3EI{dT&H%p^ACS^3$`eA7d>^XrONmPFOLvSu+$w>UQ!*y-8g znK;qjX|XDb zQAH#aWN*C|TNwU+k&*GuS}%V@_n^~SmK;|foMUf`-0V)3 z&Q6V@^4;+V$hCTm^>;hA$`UK)4TbKGw8Wp2`svW!>Q3FPAIGH*7uf{==Vc|fJ1(m$ zTn>04Q}0O^XwZSylli8XV6zM5&l)X6pMh+{ZBD4nhjHRo4OrHo4J=I5@11agY;Q>3 zdHBTbEgc&d3{jHD>czb<@-gwLnmTpz3t*F$?5=Y~Zv)w~p2BG~DVJ2Hg(d^{C7ijW zt8h?LGasf29KKx6t-8LzcZl+?TC3W1rKON^G|#p=}?0pJiEt4q}Bg&u4i_ zn4`(klg4+y%MAHzvvt#w$Ftxhj3%c_&Lp`J_Q|-m8f@DCjfUbL5EoVNLUa+@_%;-W z($=!e?6nRL2~*1wvINp&q*g^v7~BGb=W(4w$~wl5lQ`r{7BAU9mcis{z%8W7Zik=} zkq0|Y*9jMYXt$g^Q(6{JE4vrWc;bj`j=X@014Ph8sRVaCsH1WKZ48BV^_Rf|APnvf z*d%j^)ZCg*@Z3%-+QF?hG7d>ubU@mz(7@T7xw^Fx6ojjg$WvnKS3=r)7a=t=rfSB3 z#!PvMxn0M!foC-Vw8?(PGHD)QJGDR*um5yOE&*7XiQV;`fhokaYF@og1K?x1`x+D| z#l9o03`ErfNLKcb*g>YLb7hT=Bc&*HoRpb{`AQ?18w=XnyqlIX1`%|EksnhFYh=*1 zKWOS7WQ5v+r~_3|^_N_4k4kTJ@F&*Y8nG$e%wn3M$x+hTvkV8_(AH+N+r|Usu>6qp zVMx%UPuNBH>%$w3|+>8=BOW%M>mYn1Q^U8+w?RT4?Sq**oQ z3)RoJ5V*1ks>nVhEo8c+L_`@vCl5&r&cjgVC-#O4fpt0``MiT_?G(ya$hg?aBC6pb zfkwT-co0#Hu=bCTvwCy%pqOfDL7|`{LUU< zAS1GgB@|?jv@Nr?pzC#S*KM#vqBy=u^VH|;FEU2;IZhdSTPmWOac45AC#TNt?GhY` zf@D`;0Is#OvdJQ&e`M+d`QyoDmDyPny8mt$;o6cI*3nyuW+WkO(vtsdk z$grlg#&6Nk6ar2Bc3pNF?f_Y`>a8+@I(6iEEPjVB_mOQM?Ef)6MAZM#EbwNNqmBNF zNw=x1SnubCZG%UQ6M9{OXkY}Sb|j;R7KzUVqikq`Cc7XpF*48>@E+o@wW$MxeQa)& z$B}Mol4*-Y;rH>X7dm8zS;H06aQUs#>P!gP)J;M5`5?6;qXwrJKIXsL$b`3v(u!bhT%s)C}S ziOG!jFkW@`V~Ay~X_BMkvu6ngu6$2(=+1V%jM2h|XaaH9hyqU>S>)X$*sk4;+iTuH zoj2*p1l{!_p%3$(D2UGj13AIijN6v}VTfHSD-j8!HPn>|iey+o{x)C{3r?Bbu3$Me zYypbM^u#aL*!&ZG18q{j%pQ?k+a`w<6+-zEv1JPYq+=Xu)NQlBs~F~ini#vBoecRqc+fwnVxanY2e*ZXKJ^%!`l zULp|-Z_xK(NXL-YISa8TGHgQ;oVsK?1CZG~8PU~d2O;EAO7i7S2Bm&f%Q(@R$x%o? zwHDIQ?j!u+As;54Z5~QP6SNGpO@$VglDhTBm~`EzSCqF;mR|j6V{3FSf2{63zXrqp zW9MJGhh#A(N6KR+oO2+Gp%zuwl9pZV%C061`|ld@yRH+-bZBaL=scZj ztbf4KF(I3JAQO5#5_jpWXFt&JSU%`m_SvH9WIBa(WZ#-#;cvPJ`A#H&K=l<(MxIFg zH?*Jz#>4Gw5UW=)HmNtbLWVrtY6aAqAHhhF1o`O^Rxz9(k3*(pxY#}Zk&p1m?|Riy(@swK}cU) zn`+G^!aV99s3*<;T7>%=17CF9sm{sQk_C-=h9Vq#%10r^AE2qIT8EUcE>pFRi#n@F3hIF{9Ee3&y%oe=_KO*xh2egP_9u zFtWW1joQ^sW?RaV7|se@-P^^}-D92D((;LSk?YOQRbg>GxGML5oym!+s}Kcpv?7@I zjtG6n?C6(9tZ{%O2G)m25pdtrw2n2DJc0G|aZJ$VAVPBP6_E?f2j7w5mqpbxf2HBv zN=LSvF%_}zVH^E$#=AKzrC?~cdJD;+>Z0js+AWb|rP&EHB)0Zf+fm*C_ zXN!g`JAv_m9NzAM1VNH+O<P1u0%k2EO}@MTSw*Fz+WgiDpm+e>8X zx6$#t^>`l<5e9U@ybsbIodKe38~S?vWT&1*S8%Xf^$S;`4=e343En07Z;+*3Yvy46 zk2If3?wQmcR!L?~Pndc@SBvV^utYl*OJ%+v)K$@`RK)clKx!1iAgtot>RbDA;LL!t z>@}m#$Hk7R8t;R31jzHclGsFJG`84HEHcJ9-2 zl&3m2O9vu$;Zo@nyTy(IQum_b=y*V{#Gip2G!(b#mFd($s?btNnoZip-RjwV(W`V( zdv%pN4;uHc6hReJVM-^8nEpV#wL*?Rl@9rZo8IS0W<#K0$Mc zgr%MsiI;0gqv~$tN+e#x8_O)M)ygeYyqCsuOCy|nS^|oYQT<46TNVbW2(=;$uZ3Q9 z%;$!YQsh`hM$jIFyp)cS#H^J2JH#J$NvpZH`t6zNCB$8I)L{|b0N}!w$;k*m8EDjrr{)sBSoe{+q~F|bjgC*%&>jSowqeddmMz4dw5m(qhAR%4{o!b< zdRY${N2n_`35nCHex?mQPv^9gi4~fkZE01XiJ0O*t#vlpzeMENmhJT}=V7ZTaM(Nl zFu_@zvWW=73TPfWMRpSWG_)KQmInS1wIv2x)b;Z+Jp}fs#mN8K-Dtl%dhl95%$pST z03_kJ6hfNV?5(E@X_%!0k&3?*bDW~@PzaQ3iCKD7V_(ZPHUktz)n&S|=|hcaC4)Ug zd-1D)9_6$^1CJ}r+x0C}E-Akib=PJnM{-qDvYFkKb7Rq_kL0bag+vmO6h}!5{CeXTZWax`Ypl z%Ng&~&Ha|RYEt7>-H=vW zTHe^S3mP{JHRa1S#pyY_p{P3X&1|+wMtdSQt3IaCmA$@9l#~lj5EXJMC1i=f;k4sq zS`ILSINb4*MAZyCFVk2s$jHJnQFS$QQv15uKNDha)Z!dkBpQX+h1Ais?~VHrnxuSg z6Dv<&We^fQ9nn?%<{(SU2!DW?!h%UHu-EdNgxt|dxg(rXVgYQWrP=)H!(CTybwIcp zcH=-UQ7Q6*a>?lyy<%{yb7@YH?2{r@ykJu&j>t=)t(|kKM#hIL93WSLG0K~JW9sL7 zHI9%lqvMb3d7TfzL0(@JB#+BZ{eetU{0*HErD-M{3|d!Fn|7{*RQNi8OqT>TaDf0X zrWLVd1<6q$J*H+*$m)%$>t2U2_Gac5KuuG1uUIqm+OHdbO}mAVrmB?kLRW(r)JxOH zbQC&>J^~a^O$PlEJwAz^<29lji7pr2ToSJ#E2{jIichmAX#moVuLt+>Ij#Jer3*ZE zM0#n8KR%nJ8JF98(MECEIl3mKXcYC4v_5{27alK3JXCFj32Z_Hwb zZ=!>!x(_;uUrr_vZqRwQ>yicJzpjBibtwsf>9*>LixrJ6Z0%tSIN`L8W2UXsWkDq1 zm}lhHNFAa$iL6m4{3V+`yti#)x3TYt(e2DFeatW5hTKjZGalcA@g%ZomsY`e?}6MN z;Oaly1H3|u>nbLLO&p(5;jn6D+)vWHruH&X3>ip|IhMyVO;vu&8$X~8vlHM*G0bc+ zcGL*<)Nv36M0}y`5n-iL-lhRXTigClB%1a(v?gWM|9!nG@>ncR{|p}?7T7Q|MsF-0*I8e+d3Pyz({auV zJ|9AySY=`q2sex67t5?DP`^g1LZ3Ins)Q)V)J1|xwD=_5c8x?F8~X}#;zWnN*r98} zi9J-ZdGcwS^tJH=LD&Z1KS+$J+n_M99zwjy#(Q``RGWB74qaa(JypM!o05sr|Ib9R zvo*6y2o+#2d__pQET)cOV%r9^4*wViMYScIc7(2N z*~sL@)SsArBC})a1|Z=5n_6V!{`PLVnJFZt9N4iMwx>>uQfv~S2*52gPUb2|Np~Gn z^;+&$TC+*sFK~)*tITCSB(PBJG@55*pkgr{YXmUKw;(8_Z`-=y>1VpW41GO3I{X}Z ze#`?9#Ne2BFg|@Zxk*2yVay01a&G5FPU;{N9wdM(Qd% z#S0!)Pw|D`TfprSLUXu&p4a;%{&b_ew`3HUYR^aR*}EUO&8YTfiL}KfEi4Ojk9!D~ zwA;LfFTF!sLB#B6z??v=!1j9cFXo;Ft>QhyoE9;IYgsyB0qe*Qn~~|g0x0%%;4q8z z*2abtFh19hX-XD1B}X--T6&AIR~**t-eNWb9acR)@MNvg{ZCZfnyFO19dz;aH*y)& zYPn?Qn3(z}Av*bgNOH-{Ar5p)}~?J{pwo@9XV7!V0uH(fb`l!Dz&~gdqt?7JRg*5kqNDNm@wa^axDrKFGHY z)|k=A%XWFIdaRW!5__!W`Q^MU=5HjeeV6sl zQEBaxrdObzN|nGv|HeG1;f=5NJufE%%OjEg+U za&4t!{=LF9nBhuoGeSi|5xJdwdmSRSQLB0htf6=b7Ax^?`X$A@x9Y2EE^>Nry-8|a z-yFy z-4{6=XS=>-UBNUox^rvxR(zI#N7xX$rB%JF_nphF3u%bNXoU0JI?2ZwjTE-BaS^wk z5DrQ4!dA6e=lKbsmj+tZ1B?$2POpLBJphy^c>I5|dN0wekj%e9tTr?6wTjS0k z&stSRH%ADrzpl~e-Sk6R$gc4j^9HiUy#8#fD&$Cy2=0_OyT;$DKBWsS$LC2i#l}1j z9;H_GF)W3Y@R&2pw7~(O>6zu~4YM=*%<>qzcV#xw0HB(ok$irM0y^ zD#T^FPk!d`YZG_Z_qp=P7xF=NJ^LqMm9`{~V_hubPr0M<13gsBjVBw>RjUQG_NSH|Xp0C5Q^0#hBv41nkuW6W?SirP9VjTx@w zF?FQ`;$ZJlP1}1cbO$gs? zD^2;4yWzatWxa{8v&sphTi9>RD1^3iXL%(0HKsg=G!8Y;!sw%gzpXB@+=rn39~s#U z*qj)hAe`>R2o;CL z$uO1HSlWu2uG`4Do(Pb_wt;edfqFmZpsxYtR*~}4L{1ioxQUd>Q6lz2J?UBB8Wmhf)_4}`=?sj%DPP#Tdt>|Ht$VUbLjGZN}D zO%|NtkdO>m&S>^$^P?L=(;7;F;&yg4&sS(cEdi)uW0}5nkxA zCIosmJeiqUz=_sJSTkfAvK*7#-qD?@&u+wVzX5B}?cCP0Z^EdZ=~;gmuzqSAmT*bP zBTOK|MeM{r;==6!SIo^oQ<>nx6H(kJ0(mdl-lC}+R!{D!ATdFRQ`QC8m!Ax#mRa12 zsZ$HAv4i<4|CBgZB=Apk_cr1D@hd^cU-=y1bBK=vG>_!d%5w$Zem-aLIgd|}&m2Bu z$eYeb6NqR_Z1ucV*c(zzvJMIF6DQVrP07lp)a5z$c#dhxgcPLYO8})?-K;k*%B+iW z1+E1&IKPHh710}~!?`O%b)dFSTOs9n%eDe56dQD9033n%o& zp_ehcNDQ5aHH;MuyW&kaoj&*~N8p=lD#fdCAIuLtF||Z*!>z)kA4LJ`>(o#FDGg0e zuX6XweJg`THq6=@Z4!X)N}SVn_%p1|+TEE&1sDS;X9Dgvet~RoZ1A;4Vu_OoPtn8m zTjkscrV}0|!3>K@09lRIxfSB5PKPUV28M`jW5(#zBoXq>tT3g2+p=_R&T~FG%R1GV zQEHdsJSVCETL0E5LdIC|jsi>> zP?~Cj=cBNQAWxs*o*-LiCBiSR-{~I2Wd0o~G(@+Jl}E3BRHN2Zd6SxEJZ6a;fw@wq zvWZh7$q7xVCJ-G}Z-Ih{m~3m@i4j({PP_^x49931j?Dsuyc;uyo4 z@>%R4m+ADgC?$P=YJ8DR^)9w2jT|jnwv(yPW{&4j@jDlFel}4DjMXzw8$T08z?t4{ zJ!%4LjWMz=cxK}}vR0^lCSA%A>}?zWD$~ovJ(PtsraF4mm{Hr(#lV~EMn+n$MY@LN zCIHqCCu^Ym_8{N33xhq|NhVvl9Gyq|vN>q>%}l5IV6ndEWgP5kZa9Xwwa>|h3;v`f z2iaz{LWF*B8Ij!R5mTdImO+t{C=T*^=AU?-H_&SIIJp{X`Cr6M%8K!^x4%J%+YIb} zGljc!ayoP{rbOjqMD*>XN@zf8@92dvxE15cU3?#5hx}PVI_C00@hu-CIM+9{yj0$ z?@=yGNs-2x5Y{{$MWfZFau+Y6FKJ3|Mr3?ABEm@M8otLWfL9$oh1@}MLTJbVxU zenJ~D)j_wsSWj!DKrpp`#FGg}shj@>fcZ#KF<6JjA>5^odE5!zQo^`>Wt)|{Tt)Fb#4 zndq84y+hLPHvL#V3K=7;?{Ghi7bW8C3mpD zxLxk@y1pk5`*_}702JmB68QaYoo@rq(6vs^C3s1)qWBu)pWGpl9 z&X&x~k&yPax`cRaopY`85=HS!-9~Ay4ZF~6yBTZjOlj9nJ9n_+2gcR6JN7@qnAzl- ztgFAkL`l(j4X-hEidTZyiNTwya~t(ER_gG)F@)#Er|IMxolJDsJ#vE?a`x`{be%DQ z3~QDdx^IhP`qy#iM|5fdsp$Q)rq4`HoSsZh$fh3DeO#q;|F+FWD|@){%XLb?ZXT+7 zDx0ki-2r`t9@m)5A?r$@gotF)_n>09(CEp0!E(ExCIh+Kh8ql*>>H#69m*==_ih1c zFbc}vF*qKPWj{@&IZeGOy(O!FaBxdjm`nEBywHVqh>eo~*rA}VXAyP4CPi)%_GgK= z9ad%xEt;-yujB)V^xn)#PaFja8-D`dX)jP!KV7IBk+k~|0($2a#7el7qsFi|FI77S=#on%VQd;8`bP-b{D8}*#F(!gg$I4WK zN}ALw&JmiQChIxn?o7-ZyMxj3Al0*}JNC;OG=UE^rXGBhrdnIoci)jxqBiGbQ}@Rs zx}OxI(*(6hzjFUEwI7UMrQcq@M&7o@SLnB=_1k^%PW?8Wl2~_%<*n*M8rIXP$$YcU zzLUZ-m8n*B!m#Y$#P87A*XOdYZB3P;F5&;@x}JwqOAt{J++zJBXuN^@qI%~|!| zqiBmDPdWKop(k`2vrH`ut`)5}4e1EI%1-u30xwAwh8u);=Ntbgc!%hABLo25%1IbH z4GU5JNb?;I>~{LM3C4z(y!knJxi9DrW2cjp*_7t1A9ly(&h`tem>Wz?MahhafNfPj zpfSc&qk%MbOdA68#33+k;ZWg_xz_w<2mbw?UNJ`DMp+yoH|%9cUQqN&HbG~iLL#6~ zB*I6Kvu0&&ztxFl%uQ}lqQ^_rbn@`>Y_f%*7#y(Q_k=lfVJ%MvZG8yRkh5``|zP1A#mv<+L8MJl_bNUBkFZf4=XEg zx<@%fsx84|X-7ESjo&TAu}&@tuThVof{^bc@NI~@V^YrBC({_$FsWu**RnUJ9-@O- zOikDQ;q~EFBt*AXja3pmFkLKevzX|iTVz1$zbR^T_(A$QoRT=8-bBW?aX+=1PmTn3 zv5w0;BmjGIn>}$KHTBJgGA+lvXvgAs?4X)x;!09$52@L33@4!MTOSVL-x2x3~YV(}ZuYK7|SbDQ+$h%vxxyFXKF4ChTJ2!~Tm~ ztQ(Jxfd}?#&hDY#JERt~$0iExC8=C7oo>^Uu^Ub*%s$ z+=iYDi)OVf?t%tDscA}8K_|5@K`DoO#BNQF!s&8^OU|0AtYW=4u*-Y*Hco>di-y7V z8)BCwI9GOAKC+4h|H=1rk^Hi%my3+rJ2!AG)*>+7%BlkD{T{wVZrp+%#B0{#PcQV; zwbgV~W~Y81UOMf5au>2*ucyGwv|Hbr;R`IPp{sRo;GM*KwRrG4CTk?^k|;L@*cXl& z{XRRS@?8uon1PFQhWSA?o5<=8$I`B@-Pg$FmAmUGP!A@Da!NalmCb_CWU-4Afv}8MNY9XT!E8tANP~_wiJdG~4 z+UVdo_6L8=6fn71X9MbRxzIITr%ny5@^cm4OgCE!T%T6Vl~Ej+T*sce&(Tq`bsYwn zhAh8865!^!_0@T0#_aN}F}ot$TkpGta=rDQJEZtIW&`7@;K+Wu9w>R9j@yriNADrq zxW1f$2h;TxTkqu68`_4mHZeW>4PU0dGNN8;(^6cIM)Q5xrEe`LwQq)ErxDKIz>PWw zbn=2+@*Lwsd%gTPlmn)l^$m7DF0nIR8#$Z$&PB#&<;ju8XUB7>?XyWK@#yA#CU%$v zp;p4`!=?dJ3EG|KuBDPI`w-?iYHP{4Zn>^RuC7XGZ==tgrit$|a|1Ul!A6sTCz5F{ zU>;b$ZkVX?o_30<`ZM8EW`kt$`E!8QgChaPVA?(
t@@HVc(0Gm@Y1VqZuM# z>kT2n)n8j5pjadw#$8((=?#1C<5eBecQ$#9Wo0={z+iH`emsP2vpWz4-Nw3IZWrvO zVOz3YgP=))b-B2uML2fv0a^1{TE?r^NTjO0DU!P@)<5K+wl@e@Oy=igv{~noV6DS@ zb#wHZCyI6a>dOfo5XDI;&nXT^C*gJ6A~`tAXf>wS#r;^^I+k)Gx#Hz?3nXs!HL%O1 zp=8xkrT)LDoVsh9BoWN~kj3 z$JAI&B^B*4^-JMlu?yuY(qF$sSGHe6IIi|3yo=BpBMUScWdvmSYNdqlAgh>Ct>$Yqr1Hm9}3765}BBgEP#a6;|)lgxoIBllTy+efNs(ptAQB0Vj4eb~R~oE3P$ zZ1Rw$#LZOJwu$`E!v{2qC=9U~Y@3BV;-b4|#pSs1-LmcBTv5fz47GI?gNE<^x_QDK^1rtk@Ej6QlYkVMDwR#-)j?B)ro@Y+V0IQ(`>sB!jzUylq*tRH@#*< zUwYPSeP+W!zPMScpRaxzkUNC(=rImdf){aOMka4rJ2$g3n@%0<%zo~@`2rYy%c42V z^lb0L&+@Z9&RyPzdjq??w^fiM{<;x(^1O2Ygs5Hb!!x-S<&CcOQtE)&pANli zynj(%U!_CJ$e!xWh}GwPxOHxNVP9aIHzQ`B_u=nJxQMwey>Oqg&sn|GKF{0a-PB2L zleg=2zRb|OX5W^ZrC`s8rIj@s33V)ROt0{EJwa|JTHt(f?c6}?{JRJ4@bH}h`S zOC4vMGd-D0amT)!DZkTn3gjoHe21OlMow`~*m4D)S@%8|NY8uMoCkj2y}uxN&7is( z)^TMGT~Dt`H3wEzt$)jE)LHx<(|>SF=co?0D~OGDY4^=Kzn9E%Dy=ZC;Zb=Fd+y%uiO$_ioz3`~2j!GEC+% zb+DeP3ov!JgpeGj>yiB>(z?zCgbi=jrFm2@ZLOy)NJJwb2kQ| zk^=;XcyF0OJ-~Es)2-Ar&F4TZoX4c$^84OQ|d)8N!aMpI#&A zXM_E@o$omBk8}hbckx}s6l2U82RyR$>DGeO`|!%y=@imNN8|+F;&|8d)z3O#OE2+Z zo9HXjxW;&P>XSnab6nm{5}+aX7=Kz#9pd%|uT8#Z0$PZrY)qYSmo_qJk+W5YI>PJ- zUGs8Zny`m8Zs#ZUE%+Sg9aR7OeHto7W)lx#Z1c$lhAw@9pzycguCbD^>tR=90=>D1OQEY@=BgCnIK|i|7qH3A zS>&J$xH@YnK{1ne6G6gZP1qmqq~Oe0<8{NcP6sc z9pd&<6@`o@EVhwcIjHZmO`T#LkGeGH_=&$rH~6{u?ifyraUsE{B3^cQ4uz)`rrtEe z70&$0D%74{{$M1w#H$JN=yTRr0G-8(%02kbJ#u4Fln2a~Dy#ZmG71Mv6`BpM!IuV? zv8^o7;$OXo2(-b}`kZ?~Z{c~e=c~_lG72p5y=`XHXU>@wn8S_OQT6mVrk3D}Orz@$ zvuogrIP=y7YzKgQoB%kd6to1U5VO)<#+JGtb6G6&L#(^xc6(`N>n=K6mJLr` zRu`VS(wl0EKc}mHs`q&Hx|xM_kjZ7fSUBrLl&}6ydLsU7tNdi~vV!o`JQr}Vg$Q0|VLXWsGp(@(C-3ac?!QgO7A}8E z2(mithKDt^+HtKz0Qvp}AWfS1L-1e}$Q*IIf3H9UB=tV&HaXfpiu+$wq&KKLs>N|O z;>-ErWFJC)7J&0K`d0|{0vDTD{CIF~xx^*k%#(3-pF#lD;R zKqPJ$LDW$ZEr{d4Uv4byi>da2&|;;zbMPiazoBU0B-u(oefDVUQ zzakxCOZ35Lx_(6_v{inE_szqkr|y@CYoLN023a1FdbeY#*uvo%1zrTWpk*e zFL8*P?Nw|IT=?^!|9s?{ao6;gMXm|gl*vxdQx^z$J~>Bzh)2+p?jQ=gg8g3I7ADOeeR*S47k8lhYmLY6Qe63(a9GRb1F2L5%*wbs=tW@0gr$Ee>o?ma=KV z)g=?+_aUG8#jP>%+j)qoeXC?Ro^)}#-Oa=j$W*k3@9LX_^;w??897BjKqMt;8I{na)`q&aKo?eig zxJeFF>N|^_NBuDia5i<#>E)%8_TR8#xmBkjRJ@A?t37S$%~);4raN@qtg z7g+l_@fE_B!exR`dmJ`gEf&KRxu5O8D_V`^YdiSa z*VaVJt-f-;Jm=#d_u5)_Zm0i2K2@c+FNnw=OW3XLZJBUN)*Pt`86WC<{4g-b{#d+7ZR(o#0n{uS8)p`F1eA*;u8IILeK45~o$S#NBbV z`K(^qqs86M9%8WE1S_ilr3oP@xQz(DYIbx+H>IXTvuCRhp?m8jB8m$LV2UQ(5s4>0 zs9Nkk*#f2v#~^j>0lb1-Fg7=Juh}>?s=L06>c)CEv({0@iAxUuRAi^#sVt#4wrK)O z-0RekLLKIItGm&I1F07q6s;bvVGhjIetsLZi3Xvq)}DRxyHU>w7p2%C@gH4&8s#a> zMS1irzd$w9d;uYfghL~@W~UY??p{JTlriMiY_hOe(LGL zF2v9XSAsga*eWBj2Z$Ofr|qrkQPLVYPPyK_?m%7^>ZO^Gtl3z>ni)^@jqG``Ek_$$ zDBqf0GhJO{KLpiRc;FmD6PcqJB9F6qbl8t`c>E`PN9tT2KM?t{X0o|~nOCP%(by4` z6KjP8sp?pI5Jb+~6yxi=w1HzX4;F+(SbG9#xqX5y9wFQ&DGe=*}RHRZkWOhVTrIn@EKPhaNv=`a>N(LzJSne zB)#yB^t^rQl^;s*#~TDC^d$%@`z!iiUgbFbuVv2VBF1w33KA36`Z!Z%t4oo4Mn@cb z+lL*P{HhQXp8R;-QuoS5IO)T2Ry0LeKYrsK=ixoE=%!ul&98?RtSOue%EWxKc=U07 zdn1;RRzLq@7^Y)fvWkaY_p*>U^VB^{bD|n@!LKcG$(e7&W9Id~y75V`SzNcsz0#eW zWA@egQXeNO7A$B{qiTV84+3gugKOlL`Tov#oasy5o97hC{or#cDj62hAV0N8u!7l$t}m2;U9QET3~a0!jin-Yfa`=R64+hl zkfaLfgqmCl7AI3lxTbL$_x}c4DZ({eC+Q-UxgzCJHHL)N#Kp+i5}vr6G*5^ar_szx zUmd7oq3Sn_Wg1FNMqkfaB+w?i9WS4);kJoN(A_O7-EU088E}%>QfBTF&%%07-DP83 z8^#1K8{=N@oS*at;o!i|b^EmkbIJB=y4g+SM^$`QALx?#GPg%3a(G;y))cguMHR-?ffp~3hJ4Gs+0Md|h`b7#8Umu{bmEia~5 zdri8%GTq*kZYSTLZl9KJFH5&iPqzot?G?F(qlt<1kV<{aYG~pySH^k-Ij} zFT!x^+U?yWlI>7^vCG@l$rGE$+28EoAyL?(rhQk2gW)Xev8zQQtPWxVw{E$#Zf8Rs z0q?V%yVFfUeh5W^hzXIWL)M-_eL1>3==;(6af!A3Bukmz^J;Kcu)LPNwFT+`*CEj; zJ<#apOY71rfP6iHI5AL; z3;cnXQ?GHmu1sfmGX4;BK>K9*9fvl4?K`SA>2h4d3qlEB*Tt$Zz9?0tmh#H__hDK8 zMr%-5T#)fnvcep@CG4+sF4!^$1ph zt((mS9jXFK!JyAvS)QyyXH=R;SR0s^dW?^J&bOQSfnp_#BE75SGOlzxxcB;gFtF&BM$B83rgnpD;)W!Ls zUtYNi#qcp?%TMigmqjGaIz#7tn_||h`tih@DD)iC2qLZbWwGZPJVy7J2y*`9Gd1?? zeMO83eko$TedT;Xs<`v=q5qqW-6RVv?1BLzMvWR24Qe#0CSAZl$R^yvhDer(5bJeaZHurA zs38QG*lZrx+E!cJ+MBKZdau1`u~kb%6BI9qR#9xF)#^!?ws@&Tq3rved7e#z_5E8u zAF|JLo^zRV&YYP!bLPx!D9?#IaiSn&EcNJrz#Gu3FIb2bp{WJ+)Guarho@tTl^^{N zj=o7RBb|j+AL#J34`hc~ch)Q^BE{vdFR6G!kdmsnS_`fw8%x=ZCL3p@^UWv$lU(j_ z{9W@WVZGRV{eyL(G3Mg$TET{j-Oz`$Z5dgc^t5GZZMRFqxTCoE+6Nhh>v& zD(;?}*Hh>3o@<9l6E31c7E7D9MCg8BW&Ugnrj_2P9$QM}-nbl60&LVoFN%p>j_C&+$yU4d0wPK|7l3X`qr8(Basn~j$W1e z+MTRAYZM*wIm8kp7$y{&@~FEp{x@gXWYo5`2pfSE%BFM%EMtheHfjM<5*tEp?#tIS z41KDwKg|VGBl_u*p-)Bnsl~`iSu3cPJ7QB<@qYFC(wHx7P)_H#4kR{}=CsD&9zL8F zo&0sBz6XMCMi#n|=iBm4ayG<1$G#`f5Gz2Tb-(m1IIpECIj7XL3egTuZ*9G%vC5Tl zZ*~PMUBPl!aE2>bie$BjyaQXbz_nzz=Fr7!TawpJ@4Pba5TP8kba`T19!#Wh2Fe)m z{nDFW1|_!Zznlq@jWZw>aK$z&wN*M&8)pK+B0J|iUE&P5Tk7sUkqZu*8G*6z?0gQpAK<0Bu{>51Hdo1^nbp`3y?i-`dpC=R+eGhO z$;O5OIoIuOoRn-_G;lMI)~b0?uC-HSljK3n2$qQdYc2u4B0F&~hWLi_%C_eQ8@9@I z5oV*Y$ikJuJcwLj_tMmyT%5cVnHs4~d=Yp9TnTqm4g)*8Ty)Ix^oiAtrLobRgRtAd zgKXG*2MC3pi4qYtWR^Gz<*?}n;wgp&g^2?$Bp~EPpwUvtRv8-U|t%+oGb{eEyk`bA1@s(OYLxctyyYum-Cij`vi2%qDYV>GBbu6`l_?+>O>ckMdZv0!X$RVvVhrE1HO&bu zC~eMt(5`vmf6GqhXoUp>y`oIm9?XOj@sp>e^dd8YtWVv&*RRm@4u%>F5 z&Dq+OL$AOo8O81MV-kZGb&St`9`|?oJcN?U&DcPRS@s@~G0Yu4mN-4n zQuC$emk37mNm6}VgxbGflkf3$-O5wuSE2=txad?uo9$#agfb;GQO0GWSZV8f#g54(>f}@4>y|rj&3zY>PexNA1};>WR{~#yq+0 zFD`ZyEC5kV%@E>o-3+AW@njp1ghW>at-JQd&$C*J`EWM8(E3mW&maeq2Nu|vGck)f zqmf-+vvrNT^wZ%Q4$u9;9y3Gq)7DH$mPQIZkzIJNCrDc&=%8F_KW=R z25}}Rm7F~yS>A8;iekVn(gJ@O;onjjJ^t!GxLweITc5ZCib&)>97Y04*gObiC=$|* z>{_WJY$ku5WUfR^8k3V_M~A?2um}YWinL@tV9JdMZoSbCG*uoaCaD|2elKj2B&IbR zCW)I2dY7t}%^i1H1lIr9<+Mw7Z+0R1+u!~&@>{O`)iYf5QZ`l8KceUzLq*w^4}G)u zhGDawuQ-bT4KsnG_`9x`;R~CG@6;J^iHEy;zeDdU@Bbk=K&@{1Fo6dzy;5CJ2IJo+cHM27wsZy={tvmJZ?cW-J)7X zMyW*NHB5|-jTBKFH)xCv(}R~Zc`b88Md(>jn;oL{p|MNTkyZ+fj^?z+d{TIIVU-kcj&;M1w>Vj7B zr4;}J@5l+tDgjBIh?sV}$Z$Mzdq@LYnEwa^64hDg;uX~fqh-I({i4iOW*3`Mroz@3 zn}aNPwI}eHN!|2fRAlv!YjlK9FJL%b_&0QoNsxmV z^CGCmbt#1>OA7y~UFzD_v8tIO(67fRkUicz4NC_$QFMk-r-DKo%?EivwDE zeApbz6iey9YnMg6?$FiQU=?$-VmRf?bc+<>cM8eS75%VzF+tm$prw{)V4&v1mzlj* z*r8Zlybc_q4kdIV7yZJG6ok4&WwSB=}3q{zt0T3~p2gHz7ZuZ@0ljVUf zkrt%tuTU9{(k4_Oue0im*hIY=%BL&71YC8f_PxGGYH}0iGX&o>V`CK!4e^*<9Ekjs zE)drW`&aT{*r*<%>Ik0BsgxTR_@@C!n!C8^skAH%fg@Xqeua+^UhLIMpNV>S81-bA z8h2`J8`JX}G6T@@c86|(8>PLG$7m0@o@SdvgXyVkz9)zSf)22bTW(70yk(5CxIE4- zRGX-O`X&G`w@Wb>(>l(Llppy6b%*1>L(7)_Eh|uT zBi`^3(X3*$?1lOdT_+r*(!nX2%7<@(-9>Iua0tYp%fLIdfo>T=EZ6W2JBjnm&rr-E z=H(#fovJ=tJf=Z`brzpMzpjh+rbZSy^`eUaY&nFrSNw=-kIPou{}*9$P$&5KNiV&bTPjIFJa!%|f#{ScC%4j|83_0uPY>u7j5m6FI~;iixsbj*ZU)k09I;gKB5)u)`Gb7vq2 z*a74M@A~TBZ3&0W`@hD2kA4Q2vOO_(pf9#7oBvqyL&0N{SeF{B&YaIMDeWCqA;+Gw znU@PStG)wyw+}rx@An@CsvJq#^{FltdjCsSMYe@IK}D|fsA$<@zV)=Mg`)KNSc`oZ zY?8~3v4YO_b24LM;au%ywqV;)1uxfv(5U$1*$gWc$-|~=6U^WVR+z+GJ-7#;ZIFM0 zzD7W{wDb5xv*WjAi=RMooyuo~QjH)6pg!CR;v``PC?ga08IoD*`8E=YqP%7% zLE4eqo`_BAS>|uik3hhw*fwOc(}ZTXizob20jxqdp$w`GMsoayT#$QhPApD-G; zvalsSq=-0n_{1V(hKsC-pVDIbWf#WDtwK#aZ+{b{`eIw$)Lk)727O!3WEm;MrTRc; zj^p7I8~YZP>tY)!V;=v*z+33axpv3j)`7kJwek~Yrf*9f%pR@d8decM-TWlX@xCo5 z>BA|$E%(tcSlm62Qklp|dlNLxePo>M}YtGUWGqi z5It+)bBrFa5RybOXsck#z}K|CDwGnVi z_G6#Cu-^Hac4}#IvN_Lbv^O!kE%p$Yky-7|lyfJH8ufy0&z$@%gwygpuq`h~03YCw zSpRvp;u|<;;1zlOBtUPn+Y{876)T8#46G1@_@q*c>{46BvS?s6e~2U{{lhcm6xlXF zMwv@!)mCC+<_v`RBU{yi=!$_u%u7U4BizZNoRlWxQBAIww_%F2_Nb2^M~)ui6x~5^ z>Ba1UJ80z-OWkaj+DZhyoB=jP957dz>+HnntmVncfnC0?+y56VpXt;j=P~KaKC$6( zM}s?a;7zojpIGW8_7Ze*tL$Qule4D>6?+MCQiL#L-eRet8PONv40p|!k@XX>q_ZKf zq_(>uKXGVM`*Sw^Pwc;Nn5&Ci4%t+5xoF6$uR(2z&CyRo#(x}hD-p$c(W|YQ^_AvR z#ZqyJbwKo#DeDa|Ch@_j^(cJ8=1VLjtA3`1F)KTUivU$T>_jesPD{}bsZWGmm%PmT zqq43nWBf8mdLI;8gR+prm{hQ&+QSi93ia`Ud-#|1w57Q%RYb$TQu9bI ze`gsh7PNYcC0Zi3AIV@9l3!nN-dS7}rz*MA8LScV-B{TlERi;_D9n^MnQ7w%!kP41 zY(-{r8>HlaK`A-QQIyWMkAWy~V6zC(0TWyR}$BB*d7f z^CB0^(66pEo2JUpc+&-z`~Jf9!JMN86sr4@?h`)|3!AcV4e@Wz(huv2lv*7ri!A^k zWmvD!Rb^Pu>BEZHhQ#cZu}`BH8zYqdvj45kKYiI5i}d%9)uaq@I=-qLOy);ggMZ2J{t5v8zt=l&|KC#Db+15o*eLO{@D0dRW|ZyV?%e>!YonT6+bJb??l;W6k zUMPwZWg)c-Uqc!wHy0{?76TH|DB&QpwM#aZ3O$fEEB?1Oe|8(7iM$iwe#sW`u-qJ0 zyzI>4rE(n+{jSoKi(}PN{R_PO4_#`^RzFZnfltJr6SLV9A6~hsJ~o>O+`cWNrClW8 zFO4@uJp*hha-szTeewf>T|iDQV7^0i;o0jx(X$hbC+ zU24v@D~tag*RQldd{kxOer4f5gH(c*nAOVpGrFd;qhpE4IXaym9hV-(0dZ*HyP&xi zfgs5uh#Xe*T5Fd?XsO1%S}U2_R>(vw)OtP<%*Q!_W1gX|$;mADgc%jGa%C^Sn|jq& z(VHA;5@D^M3E{8d40~O}<_-_dZCZm-Q=}4LxRf=h-N16CIaNHG3m{?xHh~B%=Lxe~ zg{%q@Od{~N1ngaQnvv@_`n z8Gr?!SVp&DkJ)8po)F*qxcE@kA9&t3kKM7e%`?B-3$nyLJ&p6a7t5(;ftBNXgeWD^ zvz8;NdAhOgFx%fAu@0Y_EFBehIwi7Dqh~Rm5~VRB!g1slowwgZ?_slcXKfusGu=dc zXRiHAP7c2!fP8n2tqygL2JC!gPQI7l7|OTd`kO=b?YZt_`f-_)Z})J!_kCQxfRpcK=D~rl`{VK*dHL0$d{rOGr{|m7?R@s2Kj0L4>`$^jo$7NxQcO=7Z*g*7G~E4{ z@pm=igvYM&awp&1-w%QD%a7z!FlLSf<7}tUIObpoj2n(FhF(8aH|WGqJ#wSn*@p(i zF*h-J@x1b4r?^ z)lZdg3TAW7__$oZm0aUSS}nICeUY>>PV45ZIU^xywQf>-*J++?wbK2eiqE3rSM#cDq)dAm*2RN5 zK3*!f?8+G(`}hVer@h$j0ossR`bXhD%-Ya?EfPY=h}@-$|5MQdQ(aySb=yn6#eA9y zBPAq=QvNv*TQVGYyk5AqcC^Jzku0T7mYLd70sSxt%ZE#gK{1wq9iQ zaaiP6+ab#r*wKDW;F)%=>*0Z&Xax7me5A)l-Koma%L4~|iw;EpQ?VNdY0D4B3j;gW zpJ4U*eobb2Rwcg0Yak!_^xj)ad z98ONmM@B%kT*|qZt0r9*HoIkAa>BQV@p-VS6Hj+Xa<|@&pJ!iPFPikf@ma`#ca`avK^s5iZPnwp@ajMeJ&(>iI&B;xKZ^SbhBtEvt)P`?$eA#FDy@;o@{~ zPvkxtM~FIeL`^61zK4_J5NbYxuG!Lx-aQCub#23;zf~D**LRcDHyAxDY|iS*mWebI zOP5;iWzm6k+8DV(jQ125yvRP{o;G{3T}j?m(lQYI35ZlqiV<#aRBSX!&jCzpYjZfV z6`)(pn-Qs%^W_@uTx2l{%MtC~yD2!I0k+EoPowWhX@^#5}BpjKp4gQTGW!MJFL^8aPBrJNB2Dw?*?tMa(rT=y*dUk9`(ys=ZP&m@5<=1GF#mvKJ zC@Qv5+%dVtF4(FCizO<~G^yxehD!CpW2AGhk&77NRM-zWz#Jf*j7XOV4SSW-|F2xb z19!5hS;B@gX2&>HAJGvhXGFX@Qdg6cp6l~hFSVQhh2|;Laf%;D;aMymfx}CEnzKSn zaqH09Zj0w5@%^tzqcS@l25-dd5O>2=(t<<_p5e5#+J;8Pb)IzSed@)GA|fG%amv9w z%aYp`D-1^#G0lXx)FiJO1q(zsNRO%TZhIX^b}+V*zVkCS#6G(#)J-j;%~z>kObASh z2L|Yeiu!&jQBFx?-bj8sCh7wqLh+Ai>8G|F=El`D2F)=-DVpDAwM|pY`QdOiK?>~0)oitB7 z^!d*+7Wm~KG+$%<)PfLN%fQfeq3U^85FgN_3zmMWRDbMavV5mUT1Ai;#5pT>=DGSLRr=-Iwl=T1ajijAl%-} zq($VE=*G4e`NN`^Nd~$yZ5omzeLZH6%nxR48`AtEZ~gjON@avsGV(Y zNt~8r{ssO?hKXeONFxG>e7cO`WL4Zq4$KSGU|h+zOpo;7HJeSq+CT<-2%mD2BC7#c z@;qnLWi4kfZ-s&TaNr#%bmx;bEXcX68^Jy%(&NZ# zJ3|v0DsX`fxm`i3@HdJDq-C2pD@0g_P2lYXW}7`0OIiR45QKkcX4R9!XmAr(5Z<05y_A1F37AI{7W zT&zP0?RKck*JPyRs(Fz>klr{1frzN)vLx?ukLZ$#pT_;?9Y9PE$|A1{uI=Lw`00f= z^EL8@{rVpolcu&Z^w3+VKZ7I7inMV$d(OGb=KT_$bNHxA``Pv05%|y_@QIYXf-N+WiuV33EP z1Z8oC{+;&Wj>}8=D2fz-#M{i41atM^f2qF0KP=vLJc7C30s%X6FXbQEtiX;jU0( zU_#*K$X%p|ef2M!?~ofYvvX!ePOJ#?Tk~!5YJFH4>Nnh&ll%p(HS$9#XKv7!tXwK) zBRhIOa3%h7Vr+C`AJ-N<^Pk_RG)(W8r`>a&8>r!)#oB|34?Jse>bP=y%Kf;99wvGR z#}0*kO+1nLog77ruO&~f@dO@S`@Zji&Tfc<6B{;F=NIxRokv)f!O~U|`zwd%SS)v# zXLXpPa$>l@ZWp!Ym~BFsrZt;u8A#m{Jtc++DZZr|592T))i{`3{3`zV%C<|~nQzR{ z7Lxf#c|eK!R|WdS$vF{}eytO{@~K>+6fg1gH0SyrL9sr~(a)RNzAb;HUG(bTm7j*# zRS6@pvyCfO*jM+$v{c;--OK*iSNCF0Xn;Xy9qilM&G;tjUUet>+{ySpjCFi<`RRRp zN#Xc(C_Qst_s?U z6+bT7F3i*}0Dt;Q+8Aztoq4``4zvqNdAs0{1DH`-QXIf=wQgk5zPguldh)yL_N40e z44lHcjohv{GZz3^4)I-t5{gbs^bz|wzRT=mStRS8l7;5$+RPfm`Fea8qkjz9E%7Vm zRVQ7kU@o0J{ZbNzmK0vocc9|wTwE5a7bC0Y0Ef=z*6aCflQ+5T!i}gop1@p>M3Pv! zof}bQeX-LuuDOfQ=2mXmD7EBI7coS+1l=B`LAPBPdRl#H*U(13sYMiAf^!pIrhd$J z>kgxTJRwl$iBog>svb}02OF@${a|@~Mil*{o%zo1EBY{!+hinr&4~mScR${_7Gu4- z!;ENtPYynk)H;%jl-uy8PoORYOWy-b?8fgfS`fuF-L2SGl}lh&4FK*Kpo&`?5Y?P> zgPhunzn=Z4i?umueH6A1O;__T1y^Od9EJlGbA*tE(65l$hK5-?`HhQgWm~cBS~#0) zvqSGfCibRtxuHh$e_Hp@_CnMPEHYBfFF(`zioxZ8g_263H&{n3~eR$vaAT?!4fnkYE~`+xm2#X+muR?#ISjTwPP=wU$L;G;ydsEma^a~A|AfuQIqwaS?iAc5OtM( zUY%L%iFozfOZv?lDb%+oc*CbFA<>HT{Xu=lu~tLgwW6?ylxXtZnp{M->H7X9%$YEA4Em$aM4G3Osb{aXiq2asWVlX|U)L}c8;ARx@!v(&|e@}iTJtlMMa^6s^# z3G!{O;Jw+NC*16s7G4pe@%hv2Ox8Tf&|psyPJm1$?>gRR%G_aXTeshogBM83adt6W z#!#{JPa??1ECwDo!mGWGG_ui?6u9>amu5ZOK?5+KP}`;Pcl!xB@4#U zjUFP=mM@XMHKf>>*w84ogqxZOk*Znk^N};y37NI8(?L=9S%cO)rWL8lTtZSycquV2 zps?&xouh+q*|n>-R~fy~7l&%y!PY5qUl%fBIaDD_X)dL<7Q*A)$p***Oy#3b?C#n7onAyzD}uP{j20ecAIz_U84MNI3;Tu(e( zZ)Bg6seI6wp-KN0k{#J4fUdq-ln5U4ZJHf%oPiP=s--sBH=8?>qP=N0OK#SBR;5pg=Cs!P8a@b< z`{5q#ELwhCO4nydxsvQ`%IwWK5~3}ZtuD`5buPLKIUGK>Y`gEbTY^7jeO$B4%f0c2 zB5sK%=Z$eMjPG+!pcDIRKC;jR>=cgZs`8U{+uHLIRikljTbvXMcg$XCGCBiu$rEhK z7`r%C_f&G@V^BudBpKtUi0~aBPon5;qmXhzUdI_BGwAzmO=a1Rzy}=@BHWbap2WIx z^C63fTzGZCs7-2ueCxzkM`yK|JV>4hU}zMPXQ=`XY$5u*5emE(o7+q)DTEplIa}dU zb9vdJ+zq8-ayF+lvG~1ALt^nDVX^m1flPc`f885ihNH**?C!+p{f0w8CFq&m+mH`Jni>a2Wz zI>yf`jfGKs_H3AG-l87* zF82sxrK~!4WJ|Zq)M}IxbdORGvUL-Z_!8v+jr&kk6Fr^#NfhOvh`&yyjqY$n>|P=Q zY71VCj8C)tgTKuwZJ7Gb=6pmY#PW)#QCpcCKR+}$h=g)4_4@LPz1I3dPC_Pkc8tm4 zqSXVz(bjom_i%jiPyUaK%#*1%j{_qMU~dd3VT2oj@PwkL5dV0S0!l#h1dK+aL=pd8 z)5H^YcGDv^DRcaIK58;wRJDF~+2LzK!kk;H%dJ(V7}+n710o#PjE>_R(jTbz#Pe|^ z+v=Z-tH`0Z!nxLHjPLhgPJe8ZgzQ~ciWAN+ci7zIaF;+dm7R7Vjd!S=%SF2mEFfG z9nx*sR9!>17c?r7&mV&D^84hAQqBpLO#-al zFQ>}bZg;^|8A|)Zq3hC zyy0@I>~emq7RprV`@_O=DY$IevPtWwT+{2tm2H)mt`&vNJFgFQ?Q|*Rd!Vs_YyZAK z;0Y&K_212v+RYV*&DP=%@3Fu5!Ahj`F?_B6=wQ#L5Cdtw(6WF_7@M67hCv$ zLzR=aP5~*{{SM|vU9H$ZBKjybgcN?0C1CFdaZsl6|20F){b;2C6{>+U@rA55JYhan zu`Lt|MaMLog*O81lNmb_IAKTddf0+Bt_QzU#V?>H zFmpDt{BFb(s#31f#3uhn-xe`pOKf~0Cw3N_d$?dRnG#pLK-%EBa2`Cm?vI{J$Yrd1 zo6WN~=|qW3WlFDzoxXXK4lG{nC>%BxHyP3=ujGEy?}r=mATH-^othh)GwQ}si8mRof%BnHRD&P)uB$Ke>4 z?ros2Ldz%j>L7t7f zdiPDSypIsv3iE}ry-D{YlHq`4c$o|uU;Pnj@bEMV@^Oy)5u0}Z;p(Bul)e074^FBQ z3*4FRxnp|foZCIu(>HfC?!pRjQPpIJ6eQ2@WK&?xsk|~%ga7(akqw&PZzALuGBP0)xywmf zrb+l)6hdR3-KA~XV%gCsE+i2|CS%zASlCwF#_RRv=5IF);dP&kh51cnXegB5>J`C? zOiR64Erbt|r%U7;F6NyqRG6i4H&tjZZp7GG>dj{~j~L0mVhgQ^rboFIb(S584;{6)lzGqryQzucr` zzu6GDphaP`nhLCyumJqc$0kLNWO7wIT!Q_`dV9XGI&Y%tR&~evo|2ibvX0R)hs`1? zQdRW#p)O%_4}Icz88Hf`#!XljhisunR(v9{$DE3fp`;Fb^duq!uamr7<@bw|f~)AJ z2o$oJdV$$t*9CE`5lK|+%fQUxbGD;qMU)Ko?TVbJFVc>5V%0;MUJY9{rLrYF`B^Ou z8eM}7Nd6M6J%Y=aEeiv3p(~LsK|1&Y0VMX93^U!KWl3|>F?D9(L9f_6UT=(*)4Fha?8&yN z41W(`Vo&xI$w%F%w@TcO zH*(U)+C_>O1z|Fx)wTzb6|^rpU3LY<^nztAF|yX;jC>px@bE6NR)A+pfuAt?u#)=H zvc3il!x|lhNp}?C=CZZ2toJ1rB77_?%!K=0;{ZKb>oLzz2K_PwKUrTYv`a4s&$(;} z7mpGpl>WAu{X3ONRS3JdpG}(#zQVu}6dgd<+TlMd_D}wPIu2ibLo(KzcRrL)ObuI5 zIr6z15nFbn0=!V|0}ukf%3~E!>Dnk&4)Ci>hroeqBnDS)niC5o%(U3KINuYs_Q@PD z9LxU+2{SI{!P~X&)iSe;BBo6LL zT19Lk+VO^%JL*eMjTWRQN5`ZmM!iJj^0G>{Hn2`(Cpn3|MTnHq)Ix;ZM;F27_S$kqH@F-i*_~~p_hYO9d3q1t^#q{V& z`Fs4^GW8|t@y&LkmoM#apl0%5YDT&O2m$;gYIDDAEFmtmFZ9GvF$sSuu!W=HskcZfAdx+(g z7@Ud0WHCq`of$UIK{o(M#O0cRtK$Sce2TCZH(_y?SywFko;O}nsF11=uLuj>n{G~& zcIV8&0byWl`+Fj)BkCjzyYU0-iXDUdKzD51QbbX|Y5a*(y=3wCjOw1Zq(bud%=PeC z+0)p7Xx|-dz{}k*A5mnr@acxzZuYVD9u-PH4c+V9G#ZgnHn;WJ%`LY@I24CG(1ml6 z__?{)V&*d`BX37Lj0j%wKH|Yv=^mY~4wj%ZS;R;{2Fpz!JX7W-Sds~5Jr(}k+HO#X z?&m_U(Ja;fn6jc-DiF<5Jtys=TVhF7JZ&y-8(K|AOC~a4;>i0DT>0t2hJU-C5wNu%PMY~8 z{kD}ReKiY4Wm5y1QA?TV0hyn2y*Ldt)m`0G7W1J#x4-XwPkNGl!#7FBu1{oiM(@;N z;DOYyp*(bTA%_X>)ZnhKx5F3lf)6Mvwo;p}RQBCsiPnTa!#Pbb^pa?55 zhzBTaGx$qZ3@t%MPT0dm9(-I`M$sBV8xFt&LHG0-5l?k{DCplG}PoUC-*AA*6j$Yt8ZJvy^A)qnv? zw6xj0%QjlCocSgE!?1z`(kKcNtDtj3L(aNSrPf38gSiz?bk5F){)t9)x!eeCHeFId zL24brC?%2yAvlGRHDGi8%<#O=940b?wMxp9@yDF$MlWL3vZ2@!pQdEJI#|paR+`>y z-cyR<2wCr?LD(_&xc`!g&)`hGH99M74)Q@K!JK{#@EowoblB_!X++FvHWNr^93CSC z75XAPAfrgQS|$=ZT+~}5hWJWQ>D{$Qse-8?uguph zb~)2iM7BWn#);RwgoeK;Q925TGE2Qo)7w%BT=tqB@*!E6tGw2kBc=#;%N)J-eVrrU zZ^^r(22xd9zRLH9-l6jhQQ@XfIMR5o5WJ5+$6yA6=Mtx6)63cIKL$3Un}_R#&F5u= zI4LmfoWrIFM#j#l?meEpJhN5pmA~6q*CN^YMY0j?Jf>D8A@IXc7&1ss=f?l!#HWPK zI7Dr6WhKJnWvm;Pe41;rqXwh~P2ET;=J)pcVpTM^hlzqFm4cQ5=ZhwAiUUjgOTmwr~$1KH5COghK)w z%n=@tKhTV&16hfG?P{sgE5-2wN`7RQA=N`--=7jYjz{evr4r9%(r(<`_O&o(6QK#p zU%e_VZwU`!kBs<(YE2UK2YNTHiCvK}H^!RIyoIF|y8wyzM6STbKa0k`UyP4(5)Et+ z8i?k8i6=+z%o3+fyNmfGGtpxH`KCZvIPegewG?}*ob(R;wuJ@II`F;h7x+2>W9J?!b zW5l%eAS{3ZChS?XU^#c-N5lq$cl)glcis`%L=PV0`@#Ub7zX3}+&CxMrfP-Q& z>PgMdISxpFe6S|xDgOVN|F`r1E&l(J|31EbkKZ%+&B-0jzfmst=sbS&3;5&7S{AAM zm+E>syy7MgSP62i=UCy;Lp%=oulsl*0*|kP#+&@7w8l>1=gio#FqiT4jMGH?=S_s@ zTrE13*7PYA?{kfK)T*$#n~I~HM;o{zMhYT1d?PAzr8_1sV`Uvc55eWl#1UVt(f5Em z-T>VX)cM!V37g9)Z5KKoyCC^EpO3ns4x46b6fX1zEN7pD(KNy_pNwoX-TrrZkuV6EVAav1^|}*tzJ_bx`<|P*27_|&2LgHwdg2u5A({2_EWOH zp}#()BK&%P*ld^N5?sw+(jSaSs@Vrg4x7~3@^{jMbS4`av5&eEkIfX9HdgJtHh^4} ztgW$`IwOK%{xuR3GP7>FboYll=umJ*G{-*3E0MI@NgEmyDZ&^K4ET0BCZB)9AVP^Z z=u70NysE3-z(;^;l$sAjK>~9nqelO^8pcR4Q{U#ts&7EYOgI{=evyQds1GHBp`QAx z?qHR*1Sc^wt$Gfr=Mhdb+2~w16xotcCf4fJ3Q#2g$$q(q+Jy=dmyknAjb+H$7N?z@ ze?`9XVGSqg!Af)OEs_gwj`3G;sR$xunFCb2MW;x>Zq6(N>8Xuq;#-7@Cr-`5`^|>F zpxYDft1mo>__=wXF}Np!FM5bao2uWyZjB&6U+5++X~*JPzdG4glWg-O+uX?$A-vJi zt<^A>Z1a*>a zi#CfZubfEEp_-g`e}YRDOZrmSW@~Jl*x0sO?+!(VVrg;&FF`K3S(j`oNjA+$Hmy!J z&Fi0Y9<}5qn`S1PYLZQnWK;Q-$vKtXP5BbgwP|{?sVLbLOE!fR{aeD3ZDg0r3alBk zsgLQ5dm_6)FHO-99ff&6Bu{^^TsR{76tkRcj6UdtOQQbZj5A43b+4vSvauN6CULi! zoDH5tkf6GpEH`@EBm{V&6UWh$S0i1X$SPo!N7-WPGSq!_sxOj_3mk7gud-R%1xoeI*8pzBF2Z%c1aVs|aoq}%3y{(M zZB~s$$r{DuDjt)Ba&wN1)ykXQW#@)T1dG3q@f1O$#fiUq!TPB(4x_ZauzC40+LOkT z%MPcJb;k-g67={m(KC{998z_RwRYmr_rrI8qN+m9eybZj9kZ>ChdKIlcN3zSVL2tW z7Qr1=f_2O(&sgp+CSA$mFIunQdOvM(s)-D>c#!?qV4%^{zJq$0QlTIi9-J38%LE4T z&P(7i4^E9&v^kQt5LE4rXrc%Iu@1w8YslN03QlmqpUSHoC1LyTO|JE(Y6$>em_%K% z#2>PUudmka?p~PJSDT+>)?Xp3&<_{wn}^DXOcIvzlmk|-<-RZJ0;7uW%YbilJk@jD z>n6@((OEJ4Fz$U*?v1Z3|Sj=11|5blhd) z`>~2w{S3)qZs%L-uCpPv?6B>XH1On_W3$dR?*>debE}=L$6eK(Yd_lQbCOydI{L8f z-L#>}acLxo35+1U4f<+A10VC z7Ovt{6$hy`w1+e_J7nT}7;YO%pw;g+LECL;K&x1@-3GSFye6eY$LByf>wkyxGcuCH z1DiPie}OVI4CN*GH~1f+to{g;Uz`8`JC-V@95w&ICHC2S%vzpC0KEG=L8jvL6x(I2 zJ&sEV4ch!E#WG?({Vh98TQ3+ICfn$B(l7+sg9f<zJ=FVW`(0YSyI|Dn>b3SH6@uy1d8JgnwYw;zC>gYIvE%32L2f3GgNybc-&a3knveF@E2})3FY9}>%rCzuosA~SUJDqx z0gCeyxo^rGSkFVJ2ykNaf`@B?*MupE%3IfOlX-DgwsBpppBir3zPLG4O>dM5iTLa> zWy1*yj2bmAW~g9wkiJ@Z;!CPODQ*_$-%_K+R^w#ONXOc8}r?AIo!^!vaF9Lr3T=da}E`2 z4Nc}}RF}D5Kx;OaoJMZvrXx3%SzqSxyb(BC$=Sz<8PpRK=I{{fX>X{O4e;?YW(OEG zk{GqFJ^ZG+#AXYwe2xxq;efzq9;o*$II=cKv}s}g|8CFEaon}fn?Eq&=3FBcpn zIXECYS2T{b;hFQ6gK6DA-(m>9g;_r;r;iP&cogLSde#9+Os=a)McZG)F_N<4UZ zjQD8Z#9f8&3!kGR0KN;Ds1I{}Mn~WXnMWIRLI$>oEQaEp9A@FKIkF<(P^NEbro!~4 zl4yPj$0q9%$y!3z^!W+%`PlK|knRArjBrdhn!9tpuD40lf*b6DlcnJCA8n&r+bGJy zRZu4|*g$4SX#+$;kmr6GXx~luf;y>`j8v2PBTQFz?aZfrT5xWyJ(H=fr^q1p13wQM zu;_RL7#y9M!k~0rjt5Tw*Ga#8%_QZF4A>=uwWoUH;dSdF zNZ!twCr*p|>yZdf44Lmlgu85u7pHFaGKAInYY@&K6CQ#yI`@gJiv0Aad6-!qpCtDK z^wuRi)CODNN}qs{lkbsj2%`yejxrWGok0zz-wW#l8xOC!byZuIz1Z50s0)*g7V{3k zVIYou#GtDk%R_g^)#8a_A$~XFCx|duJPlb*EN0A4JnZYmiNnrf>2HZ#zZeS#(?hR1 zxv>&}jFxQ_-}69sVyWaROv?rgEnu_xjg+UxXHS$5Md?4t2ZZNl)6YyZ@##-Vl`oVy zRh}SkOM#Cjua%#Oo8iVxh=~?$6q}>(oJroS0yN7ee>zuI5{kT^l0CH~<9ep4=U5X2 z8(Leti)VA6H&={hm0O4zf_*~}NMIY_1GA%Zm3l-cW427j`NEB?{Jxx%nTiz+Y*Bh# zcx$eiIVR?vIWBhg%#&m8nHR?=R%>oydYb#%eL{%m>C1E^`*Z(`hUQp}*iI(A5n%&T1EI_MRW3D@l0&aAQ5iCqwCpZm*F>(-oqNT zSz7X8A1qB(7NkoFS3uH78QMZ%a2Q&=7=cHMt$RTt$T&pi>De}svHHup zr1`*+I36jRP5w#Qc8v=x=k(uUk=>}fL~4E5-fVNLfe40mp7b)=bq5ZvQ-QIux$H>n z0yH1V#f5AM!+3DSNc*c#(H8DXyqGec%yzrBWlmGGIsR8lauAFrS&IwHb|zOAx<0UW z#{2QJp-Sf3O#1!o&OWjYJG4lQYfYHUB5OoeBTC64>OGE^h`bzWL#1^+Fw5Zt&T*#7 z7(4EJKF7l2YZ;y6MjKD}OW(L)BNTw^Jkn{2^TG4^_LqfxK~&KlpR=4qlL~T*CZuAw z?~#JCM}1o^hNvY>sc%bw0dGJP4cFKrS4oJ3c`iS4mj_FV6by_051dW*o+Y_Gy)!Sx z(Lc;(=9Mw*7n!DxKg@=LIVee{T~cUF$LV zLMmFhIQ1~Xfwc>09KS98JW)B&dGprcpWQS~j_I;u9hfr^VY@)40Rh-BUlR3Wuow6qp~`#>)0R zQje_@3a^?pbbEYYu|J;Q5FZ^(^LQ8x3kKG@*Su-HBOTgGsYJhf;FqAE+`jZ}5%Vm% z@jeWVPOu+e>t1`j?}725aFe=IR3(ccY+#&*5E+BtQ5H+G?hI**Swp}DUxCj##bmxO{opJ1ILrf%ym|s zPSmm^srpeehY@5dq2!;7X43y+cXTB;sQsbz@d70Dfy>k~itnqGC_1fI8QCD^j)s?pXTr5jXtL zw$#bN-~&6r!NlO&O&7)*5=3*V<^Zv*0k>fW=i_p8xkwnv$)5qlHS+Y2OL&UN(^h#} zB~J-?x=@}L$?WqbfMyusq(1EG0}by2;}u8Zo<=@gKcJqFbZ#J~T~A!^p5t1K`iUr5^KB;< zwxnaNC5%)&zdG)P@kQEMoh?ET>U5ju)or=N)PZlYtk~!aw$;m?Tu#@9&LUKC8tCx0 z&%jO6lwmPUFsc&z3S{XEhj8ppD(6(Kw%NM)NMYiTpxjcEm3UuaQ8dS+5?7e#DR!{yo+Ah$|Gi5Xr%u9bm4-=qz4axtG}7RPCOz6XeEdLD*ik~fzr=Jm9p*UrFM@e}RDi9?B+$944Udn@f4 zuzPo^|2b@yBS|LeU1)yU0URdw6-;Xi)rZaZkwXHJWqIrFSo z>C6=p^y1O7I*)pPAe2s>zagX{u@>4y!3ogEDYJb7w;BTOY{OZSV!nYwyQCeEoDn`w zL1IdgMwqDNyd#AWs;1carT44_fb_6c^hVD ziyZ7cH4)X(hcH59S%S@e>s8V+_e*woE6Sr4wn!H;7t;*_5H_;Sm9N)hB$3VwyzQ%h zTP_*n-44U8?I%c|5R=Z1?4TFT=KRN6~?~l32et4Ve2?eRcA_iNPOa_ z;7zs;*Iv$$7gMUz3{*iUxv2?hjCc~R_-UANV9GIJ@!+582KTP?;1{IMBaT6EJkqO< zM^tcg;e~ssPLC5C)}3m|YzBuiuJw}}LgtyRGQW$xL{ZPwAmI%kEsGRH@L*2csV4n- zdbWX5upJK13AgA8)D~}YHKdnu_G})8bFz*OaP9+#eByTwKAf58C>k3dr=hC}Y3~md z#HL#7iXmt(Mh2ZF`1eaBGu(D;*;;?^kv!L02{5+8_QENbBK&n;x|P>?Z7@f>Mt}=B zbs|DmILe(TFUkOX47{*N1R4(}JavOXmxe4vcv({L{F{x5XL1j|-}~1*SD3?}N3-{p zg&|@uz6UU3#S(XTTg~(P9IsD(Lo_RvmEEm*oKx*4 zeCvwY9yiKOPV8#&%`>4PJ}Yy+)F!ebByu?&Dq!kxa4S*b1SV%q^%Z@P{i^-71F3oT2vv*v7;5?$Bu zmV^wtzQ`5#@jl0&McEtu)_eetzoESmw58!e=}R4_taE!B3ZDNfc)H+}CXdWz$S*OU z<9XE!{IT)YQi4s#|EVP@XCLSh0l5V80S(Al_9HO&Kr~P1OfFk8#+;eS>y`b%{7g=A z74OCULG80S150sMq2}JqGYZuqZmMU3vNn*pn40J9b&O9qBED}t&RXcT`Z(do>6EJ% z_XqLnAlF#=)c#J-!h(w3WowJ)KL6LiI~%833;mKa564P&&N<{P3B1$(t|J*h?)vLJ z%)Gcpw%z``>1HqsTx+>Jn5^ee*PX1#I+@dPB5c;ApXhxv&zg6RKUu)vLX4F8DZ)g_ zYW7=R44M7bqI2D1&%Eed50=j9or)+>7iX$t`8^8?SOkbq{-x5lzpA8b-^XUT$cdsC zYBv#<&{v-iM$Qt%i_c1QKqU$yB`LH5$MWKeAJ5e0Gqc+Sqi^&@D)}Te{^w#rT{|A< zC>8_^*5P1@{j50=W6WK`ah$48ljY%0)g2Io3ia0w%5acLaO!|mD2BvT;!rs%5jdGQ zuX%JtHO17xnJgXiQU|n3f!S=A%@PD>VrxAOv8IOD*$uJ6hWP0qc@3lT{61@J!zptx znxkVb2eYO9<+fOtuCQs3EBG-)ht*U_Jxr0-)NqQ6X?M*XvZ2R{UGu zC~<=}9}@Z$zl4+R>c+x#g^6I{KF;#lI3+d|<;2DXde`rlG|UfF_d>Waf$COzW1_*!r;b-Fp(373U-_^QecbjsEm^K()>L zZv(7hwTY1NWpeE@*@c;ACbjW4gr|!4lU>4Q2;6)eow}o40OkaJZgH)(GkJfESL;Y} zac8n_3kFut1r~4VD6p1o3B0mtG5;$qb!uqsgc_ASZ`E*xe9+Z9*Y!$rzS9@ZUQj2e++NO~C${_MhU3aGL7KTVcLEL_xT2ghlg);X`$IR!+ zgJI;0u8rN6=1WP#HLeem%eG))(U(j~wq#u=?ot9$b2TweT<2KZ)$@1YKu3OZ<(6c; zGjK5GckNB|y0c&U+h4G>tX0>^kd*CcTAI0E#xHZfjITQT))NK4eQV@rGlnD%){RQ& zLe{V|F0fV=C+iNtusLVMN$xlsm&9V2wT57#m=LjF1BZO6E+*FMPj(bT`NRg|J=7Cc zkW)%;qTTPwiJh%V62Z)gkV{ZSmMk2^I{}Lneiuz}CkYg(Po89hr#|th*i{4$wLb_> z)XV~W-}))s1MhuZZa*n|T&_0N5(R~7Q@r3lfl3dsKYacz6sO6>Tua|jW33z%hYfj& z?H<=2oW^2tQR|T#qaJfAYu8%H{dMBhSql#f1Y>C&_0@bPSIb}jG~&c8l@TT)1iY0O zhK6{|t9Pqpz~0ayJR(fU;L%ST*yF8!vGYQ9u+6;`r-g*&FEiH7w(x8ho)j> z?{|ppZDCSdk%)E%pMMi>B9IlVpQ2nx%|N6p6Gp-m6qu+VgC>r9j}K$rbOhr=*Q1F? zOA?o`qAw}pzb9u&e4q_xw3y@=e)H{9`fAeTETg0z*U&X`gnHPk+QE0n z@*}x_J5EZ;mmj!!Li_l|V`Qqu`RZgjB$zzGKJ-4+vCU!A!g+_3m=&35UOiO&*&?S^ z^EgkK``Yb1F8X*__XVhb-;BEpyG+w!cK@q;HF~U=g=9=|>}#N%!uXsF*C_ul2;@ZTgoDdy$Q7 zk*Bpu6{D#_H+m1j2J7rd3T`4P%%KUE5krw>rUHeaSKkzAT%K|$tBdDmW<)$zWqkrV zIqxE8*| z#U8Qr@u~H>+WxF;`_J&8ja%`<=y6_aHoF)(>zM$yoSnj&`K|(a^}c4(#Je4kX0K ztDx0=#lRvQtOEq(^S2U;k$i&Iii1|=Gf>bbf3%P@ zWLVGO_&|+>h)w_#up=wmNw)1v$h0Yn72$2YeRq7qaQ+1O8P2Sggn)OHiN?57hxpTo zbsc941+-r>*LK&Y3*euUWIDOeCmp~Rqhp+|dQ-;-T9I!TvAA`WU!*PLoZTU<5Gc&w zJW}lgjJ2!_Z=MvB<5!~lsC)UFshNI%R~zu--0)U8&ikxZ?-zk{7fZ)6b;WLntBfKp z3wJyVwY%K|8dydH8Zt$eWAuo$)vRjyQdC>x5G@ z=4_sBw|$1Tjkg)BA+P2GIb?{wqHS=S2;{(wPA%a!ktWYt$!EyRix?f(9j+I+@B(Nq z9%}V6ZA9+bpD)5iB%~vGieQ*Jef||_uEossp)zM{86b`0f^CrX$z?aY?`{Q0#!CDd zuxP2bRtzUMB0p@**8aGns|>@1)V@Rq>q$DmU{UfpMF_2(c}rrUsci)M+x_BEL*!O) zNP{0o8KntSk?naGm1u44@*t@#<`VLYlbM|yy*x6RJR4A-9Rknw)Zqv(|KvruA3Ci69ARj-O8k@;aS|O=-x3b26^!a_r;J{ARPcEcW`FkI40`_FY-+z5_fX z%6)M2IClUTWyTAexG%~w{{?%-*6iYr=BDawR`#d}rjOJyFe7%L*^_+Pt6F<1-ft!B ztt#An>kD;f($vzDIp5rQrwpE-(~tAbGdxSkZA{4Ib9Bv4YF#5T)1dKNbfxl~RBYcfa>c`eQu;$T}U zyAH9LW$8`y`mt7C=2#m5v;HzI79)YKL&W?cJdM3HN5(DZA z>AOE!**zz)v;E)Mo_7>yR+nHgL*r0J_6&@T`WK^j^gg<5}aCX3tq>6`CKR&#$3i=G^D%2{02;K6shNEadF4U-E~6M|yYc>Lx=@^o!$ z@y%lu)R&kCdF6J`Oe1Q29%@V{d&idU`UbP!ewc4QXFn`3SJ)2=O+p$%mtQiBa+77z zjcSr>0)TeX9oZ>5&xCmf>9_-8ZBH)o>RjWwmTe(K|7@z{=hr&zx$~t|$qM+-xQNlt z-bG6SJ<3?9MUCeG10vVH)PXA!IYyDTZH~u$Q#P8alJ44e{lv&aloxU?V#xo;+q(cp zRa}q%yV*^$z`_PdAPPv7plGB<5laZvB(MQgFc3*Vd04BMrdAPlK`bGO8(Nm@Dy_BE zueNHnt+xHPv{j3Ung=EUQG@Wsgog%A?W`LiJcIy|{eRBf-8^ilef;`IH}~F|Gp{o< zXU?2+<_y7PV#@tz3%Mpuv7Nbirzu$Sm2_CNWAHRcR0sAjP>T=G*K096W}(WGtY>!2 zm@On_{5w+1)J4%_0Mc2Sq&bT=NSW8D*k_>UHeT{?7SPDGnF5C~k7a|9Is-)n6re8G zKHFloivw`G!(!4~U+d-F#5Z#jg~=&c-XeNE`TdZPzUj0Cr60IEa62EA8*pr+a2i_gs!&9E?By2lyo}ZvFBg z_MFBB4B#75MJ>aPcB6Bp4GH27eo<9KUZ|d)&TVuvxh!!1ciSLW2a-dn5*^~|!1?u| zfk#~nst6_nOQ;|I-DN%C!@)h>&}<8)e%@s1POmrtlK~_qKqiyQ6+zL!i*u!Cozab0 zbiK721r83qWZV5Ya!vywhd;+b7-u(fIL$$Dq7F{<=OjmhQ*>~u4o>#xjEMx}ZWidP z3HIkqK){j4A*A*!rZn4xjC|=OP8@OnI0sdHm6fnVM`xQg(kuxD%Wx}XnMYGn;7D7O z(xY0G-&?gP|L?0Mb&8aqPWkL)UYQdqKV8bV`YIbJe;MT)vr<3`6>F1L#Q5zQMvmP- zHN$S?ICyXlS=TO#VTYQ1Qde5DF~fAQM)|~U!D!1C)|kAI3?=E2VT@AY&=OIjK?8J4 zbriN2;S*Ftq)lB)HMiMq+;&YIlYYZfQmC9AIT+!w(4})AP}b#H;)((1u1I~^XJ=$P zyzz(zz4}HaU6_`>6yoIcF02laIU4Wo`nY0IoOhwAFAKl8Nf;$a+Km)OHCQhw2ve?v z%p=@`zRh!W#r!z$6+NM6s|!tNKob#ifzHs+H6Jjna{egHSGt)o;FlTw*L}{wQW!U9 zU|s?O*w6^B8{+V{UfUg+^`6wscRjUfkbQLKF&W>m~G5*waUwUtq5i5qMgsYD7iG29A=rQ^4do1NU32 z{${s*kA0=T27wd6-z>amDqgsaSpyw#P1Yk8PnQ~=}!O54s>F}qySgN3Hr|jEYukXKO%s0A*eR6 z4_Q+{9PW)6W2Fwe2mO>v*I-abaC;S}`oF;G;=Hsf|J>HhhGm0`Zh(~Lt8Y&M1#?@4 zm+G)>_Z?2irBGS+HyI^3Ds6*tp6#o=n(8gV_akm|yjo;Bvja)*+*ag-Q(;p-WVDn=Bde7$QsukxFvxi7<9I?SckUb)Bb&UcTTwJz1DF`o^905K~$Qb z)~@cZL;f9)?n}5+(g-rdG$t#pma~VIS^k+$-YA?-v$`H$N*8u9DCR6c!kFtwt8-O| zWO};CGu?CVbWZ^+F}}=*D>5>?X{dJGP=2lLGWF9oq?AbI22#Szgfy#pEr{Fz65VY0 zTBmFC3Cmc=>^6k;G5$Cqi1_J)9_Bhi4^(|RKt~-9eQZ?21u!TrRca_;-j0*i z;#$>Jv5=V-%yAGA&Bw=qOW&snnR}Lh9-Kt1;cg|)n7ScR4vAVx&DudBtR1pciyX(Qb zg-}T2Ph`bn0D2prxO!ny&f_g_4m)YE!C(#k0XI7g>~mMU>d;V`Ro)KUp3dsek&fF= zJG&a!6=}HDoPaZ(9>0WM4(cP(bnk%Cf{i>?_Y-R7ZJVD*#;kyRiFeI2&4fl2Ey{1l z0bP6D)PWA&@@dJzOK?Zl*=q%KKRa;g|3uyY$r_yl1nKpn2s%>!iIjzfT-7Ibo43QP zWwiEf1vLLA`jUwgY=+gB!8g=Qc}mil9R|HD=`hn6@A^yHV0K;Z`Usw8()@FEFG?MH znEME1(HoMHmRp`FbwNz1qBt%%=)YX# zAStrH+1UBk-~eV+aeVMe8QcHGhIW=pLl?yaA7g+QJAxPgw@QVlv3bAYyPv-kXlfs` zxdUf5Q+mB-o!yP}volrQI$8KK3FV!44Rdf4P?CTTF!7oCOVy<3lpMy!p?9kZG87705u1u2j(LIA6yBlQd({Bzl zO-?^fKc*Rj;1hAQ#oa-2+k3v%4e)#e7)jWTtTI;l+w3}(Jhu>!zo-}M~N`2 z1+y4yUQgb1&(+1^bOAGsIfWFWl0WZcJ+6^Lylq9d%Z3W!Po_z~MiLypO8sRYbR=qc zqO1C#P{0^ybOiCy$nKX>X}~E|#t@3l+>Xp?HFQoIdg1*!P>MgoP;!D?7WV9^nb}#( zIGz3`+fh8jATQi+%vH8+hW#DC{VHSXJMidNVaM_gO!`%h;P)Y4WEyb(FgS(2mlaZa zTu4-X)g5hitGBI_-b5&k(2jpZWFlnNq%rgy9mnd#MPJ^bbD+4tG5LM?0kPi*a-e}5 zNrS(YvG$K>JWDFn*sKD?9`ZHE*aLYtwwvo#PXgKiR{k2#4ln=zai*2Wb)T|;Wx89? z99vm60?zXCD!Zi}2g^l|R69hh0|Ss0qBk*%xVL0`(& zh;ezJPBeiN1lsRzJbpH1NOmU8nA_sH(14@e-;>4YSsD18U_Wq85#0vObqr-ey_8vt zB@=T*2Js+si>t4upAM7AW_&y&1LU#0Unj+eE_+n-tP{wS`ZrkH8bf9z5 z*}Ub}*wNYaF|#?uBRGO2Ow?H^@})gk5uVRM+2J5p1V?yv^KmxfbG(!xf0$dQ_R-T~+_>E=&Xx(aj zk%t)>$|O#Rjwf->95t#d@=1xrwp@XEnAj(+4 zJtpmuc`I{N#@Qi;!XM4r1vfwzjNFs}o-wmijNDXzHb<()__NtxCtwU9CP&#=nkDawlMp+u!Y&})U@Z=fE7jv!vOh=;j9M;bKWeIW+ReILPDNaD_8sR-P!e61UfVW z4eJc9V_j>1BkAGF6$Haq_0D&yuY&EuIN05S72}!#!o>v}0huc&S;3y!s}C(FnK=U8 z`{kTi3vkyp0XG1!@XQwA#j%PuVVNc7E0ktmeZCPVtfFU-u!$ZgXH;C*aC~|Fl)PY$nFT(daWk+!J2CQ;W*HBY+-i><=)iQm?HF4xME8_JOZ8rmjm24HB%g!J=`Lpex{yK@Z}qwo`dhdFw@q zfXaKOGD`OQCn1?})J?TISO_fsmC4%MN0d+xpBp9AvdakG!h!|DsSGKZW`^auH0%yb z9CU_IWxb)ncjSyq<)e)zxA{(oy_sws;Vt#;k<3c2!pN*HMY)e{s7q`g%abHN7z_8H-L#-tE4FGLA6HWZ1MIqgZ~G(S~>E5|zVrLDobb zB5TTZ7$+&uKlMt7v7IX#q-Ty~u5>K>KZAnYN+?t{GXt`LkKr)0F4yUsmZ*X|kLdLs zD%KB}KyI4`lNyTuKa>YQyiTLd-41DE?tKd^hG#nQm^f1C(vL{qIWK}0@`X0JvIEeF z*>HHoglx+MepC3}!EX(}r}}<<+ceds0Jb%pZvYc#7 z1HWHS$+o=2@3yJg7LlaS_TAATzIe>Tc`}#1*Z?IM-zZeye~-54^Kk7DOAMU3I~7sZ zs1w<7R$Tl|NeFKgsaG*KcFg-2P!U~Vlb31Ch0Y|h@3G3Zhut3AUd%gh;ap+9nj3}G zk~&w>aN&^h)y}KT^`g`f$BM?7JekGFIt2%mS;sm@~+f1j0mjo z4W|)j%3IMemx8#}g1C{$-4?d1(YpnCV8lQ7xW?XCgdAY+Vz3v3FUDN3S1H)D&Q|At z1T2abRFB?U#HemAQXBCl)g@E*!*s5K@+~v0n(`H&m;&Xy9e0UaXK9?w=OhYF)YU9gCC~O|hvsVmv?iCERu%lE9b!eGWSV z!V;L-jsS~eTC3MfHDD+>h^|$f?W~2CsCXSwW~bgDwM)+t5k3l0w`s;`k}RoQQWBBs z7qxP#G!LsE(JAcDQz2LgG!y&Tga7hH1X*08nli4EwU=;<9ddXmk6zhHMmnH#;Bo7hh~&GR}3PoZyLlo9Ahll+>u%1aps9(%$pM< z!=~%Jn7G9kLrYFVATO6OLfT|~Qq41PhQGk>9}u|C>UufGKg+JKM(EV;BR}?CXz_j1 zj&CvKz_1T+1h5a0t8$S2!(n=pFe#!}Ig1$f+WT?%BODjzOk=Jc7x`vZQK4_<8pnz& zi&N%hw!3oLv9I*Cf3b3KA#(BXH6pN^y-MmHF$4WRAM+u@+>RF)nFLxJ&uxmHE1@6x z3IJWG=Tc)_=2=(nNlw$*QL3mPqg1t_!+#~rkOXR0Lp;z%o2`yUufJjlXxxX&})_@&yAbP&?{n+vLs_&G9lgrLJg1X%c(6cT7$6F$n6lfY9#SS&`D#`FFbX7W_kGfO>Xu>V>WGckx zsUh>orn4f}!fU2?KT|&rw=-Ac6tO>PLSGYKx>k`} zHe>v}0yCNU^J|Pd6n?sa7WPEoy@0BBuEcqryMrwxwx8-rsw(;s-{j_xrdeV^tL#jQk9+QNIQT4{w z(H5*vaM9A&sG9#{YGzNZ8EZ+P^_Eaj3M^sIGM0kS1&@DvFRe^k00Kca-cT*fQ24u~ zEJE^BEMJTz2BTR}~LEQiqEBXdl=Tu;j?Gg*cyCo;c?Y)^)Fbg)^d+BvO7q5 zRYi*@6`^d{;u$Qm*{~-j^fsrAq?EL(nn3p1&bKc*-#Q|1!Yb&45FF_BoF7%bPx_jS zBYIdLV;jNBf+pRWr#=T$uuhmZwPs9iEGW?aoZ*)xZG7WcotxgnZISmU{dA@S+Nk3E+#EIz7I2?Qk(^gcmx1ic&j;iAPFJdksi z#3cQ~e00b|J@}BOqj8(=s-9-4@l`stPFA%D5t`|k?RTgY3g+0Nl7Z?^2iIy|G)GAp0|4`@og4`u3VYIjX{N@qqc6P@+*)zDlkkH0*EM zf!DfOJRIj2j@uU~X(>i&>0j8)nU)M(`X(uZbaj>?l&`)^=ZpGtCQB%kxdnvW)(&z@ZjhKo*bRP#Q($jI*ML6-<+=ly5{RDxl(=A*^nc!>I)1&R>=f) zM#>nm9-Wr8;yurjbweS0k%B%A#rpZQYuaE~Bh*3f-c9;&kQ7%}obX(N?n$V_AbD!? zP{)NHnj9`MUuvhtSQghrJ{H%ki^N6|Ak%~DM8>l>%ZQn1^y%&j=Z4 z*)iS`1P+||O`A6nD#h5XXz{*hpxv(t*~n~IrLW#vV}wsuGg&R3VL^bPQ znv5Z9ttc|by*Cbg`c83gyLGHCn`nh{RA9g0_Meby;F!~bj+(&|t-uQNO+OAtd* z?5kB7G4qQMwpFSGBBSo;QlTxyL)HBk$3Sa6vBfd--@f$Owms7<{{F4H_@~3EnvDeJ93vrz57A z+L+@DSyx%5EN1Q zze?pzra9{qW>kSOURZW*9QZBFe!Rmn#$Qf=hr3__CI9P82$+*B&G$}}Ie7!}XD*8j zT{2r`%PWVl&NJf5CQX>k*$e>k)U9SyXrc=#p)mmalU?Rf2_r7cRu}rb*jZF2sQuxO zXbI9d&?%QUnCMJYuRt6kE7}v!3kwsP^oCrk!XE!gnf??h2E?8{H0oCpAUhk5Juom> zio0Q>pNWloD5l25!9j30ziVvCAd}HmNMBX}g^;lLkS)!i4nx)o23F`%T}+T$I)Y2E^AHlHSi)VG?dxw7BMEo#ujuL^E+! zh|#(k>iyKr#A8O1?8GgL%%w@MZE*OWr_2Q|F1o-C0mJ43x1R;>au&D?A`6_;T;RN0 z{X;{^2)o%6@h18-3gpd>UnE&Eu{I-LtYiG_BJ6q>K_gsiSAx!HtxbXiPPE1nA85@yK-195C8Mfq`Y7o?qGwPhYP&e?31E-P> z^#%2jKy`@>P`Mz%nI{8DwL~LEG75}#N-)n^0E>q&Jqwbf17CM_ovNTNH3opB=vh+D zGq<0Z#-&h+ix}APn}phi;xcPX;*Hv8B?g;WC6}l?dn+HB-M;{PME`h+Qw=>LVvEn8 zx2R^^d#@~o!}B|1d{rrdIrj2XZCj^VO5&f93_=QeQllQe1h#bYBw4@MT75>9!{a4d z{Rs)GZ?wEJ95+cmnzLjY=u9(@EW4=uYyGrTsfs;Zq+{>Yy;{l#J_pXIcak&G*k za#PqT+{LPH)Se6u9E}#5)UHOct{12ZP>u_3{=%OkdWaItoP7}27pkRf(|t>Z zGx?V|IXW!D8U|#u49JOa;6UpU*^RIyt6RGY&!KRmPTd4(4Hd{j`A9yYR<1N*ag9)v zYc)lYffpTS4u(~zj!YS2TqP%KI1k3X@VP$|On3NG4|8SHtKGl`7T&E1Jfp#vG(;Rw<`Cwe-1e^J~OZ9^c3ptc) zar&#=jj<3N6KyD$Ht2=8 zH9?%c!AmfkkTYU7iESe{9V|@54I-BddCyb7gHl9>)I-k*LLyoK!<)>yEY_~$-aF|{ z$ZTAZic_Jm2PK$OlogMDrZEr9mM)l;8|8fdAI_Z2bvmKH#)n9FXTj>fA`&icm1m?? zd<@zn@!0l??n*T1<6Z08iO6hR9?MaOg<>R_S>^fxQx$2&HTbLQP%l@KBkH-W376*D z(;BUDp>Jk3Eh|EHe9^e~%Fi$oN>>cnB#)Cv^dl5!)SNuR(ZuSIO9z-hD-k$ny4zRY zflYKrf%={h6{O--dQ6Hks+>X*`+ABg#NBbJ8%50V&?N;b@eLp)Vy@_2w|P=AUt1TG zb3Kj#W}w2QrP9QGuC*hm1R`baRZt@h%WRM)YtABcB@9?K9`aIWxrma270hzqEN4pE z8BVj;f$QKZGbAC@Dm#~m;k>h$BXkZe2tZe(M|_ry_}}^IH6cbBB?+D@GfEOYm+QXs zj@3nT7LH>qvf627j8nbJF||mC|LiI0aP)l!b8Qwi{=^MyR#DisdWSWTl$lYzaAB5?sUg&Jo9G}9U3ay|RH zakB3c)ms!%#C>Cf|>5~8{D?`UC+vFk(*6mxVJP?hiCNZ_gt#@U=E!HhB9 zI*|>v9I9fw8t}I#W%OxVWqK>~L(fslMkB)CGS4Ucjp4Q{?eK@=!f#4z;<%67+sC)Y zZt>RA%~!jNQvzjvzIkI?=BoF75uYY+Ce=L1rDw39%OF1)mvY3O+do%y4^05^0i+l-w&f z=BcDl30hbqS|(XwCA0m1BZzT2Ge4A0m^GJi-Az&8fqgO>Swa>?1iD!T>aza<0npbG zh>KUeg*sBc)nA|mQTeUh45xlYK$m9Ww|t>l2z{g;>Jk2k8E({sws&rycW_6dhrRO+ zsflc7@Z@FewAJFE-hn#ofg&3=xl#2){ITbPLkTE9RmSLdJ@AgK5Ow2fUts^tC9hpN zh`sjCR2G--*MOP(7uVWN@~*c|uC+hq-L?5|F{m}-d_j9)h20+?D76!0TR!lZYs~;( z&=FXH=ea$uK&j)TC}b?2Z@D&qVsrWa#QPH0J+qDnRvhQKD8Uy@=b4+rpS#loE7ErIymd{rCyeB14nqKmkzzS@uBN?{`RrZVZ~5w$w|sWy?Fbc++pLbjiVh?EmHFCxO6J#G*7hFilPXfMjPgqtJV%p`WID29Be9Lp{{_RDbUqY}HpsM}2ETj;7~ zaI7|4JtqE6NtPOLe4BWl4+`9H{!Hh07r#+ExwRmEeXe!gH0%1@81h{=4a*>#uTJN_ zqceAK-3hCqtl~knE=KC`t4^p5CJcLQsUa#YV%K%XlE-1n+|O?}za9Lx^4r4iMSj2I zXH0#l#Xt4o7M_Zy;@Qd*Gp2{yc((C8&ht3WcAgmcJk-InBVdP(wOG(Lv`5fkgr(65 z{D$*O=f_^y$p(Idw7C1b+4@Kbb?ku=&=G|Vc&_XStj_3&bV6!ZC-nc~*e^RM_s7MQ z5k>Xxjt@@8yqd5g^@lxWb+-Co^?G?zzW+f1$j@WLV5h=@w=5@5x=GU z?&D|3Z|XzC!S!&SoIZXynJ2zUAI1ypLkT<+cshAHd1Bx6u!AQyPY>IB_Kk_7=g9AO z{I>9G(0NDtCf5cAy7h(rrnTMrfwlo-hV|?_Q_Wat)cTrT@R@7hV@QaMytL}mc+VXX zg~tZwHje4;nI$x)KnT*Jbmn9{zZ+$877|ptn4o)>Wbxl|O5i5^*PCrS}k4 zx{|Qc(xriy%P1xAhWx$G`z&1RjV#8WigtKt;0XdPB&EDkZc|QR-e2cE>^tc4T!u?3 z^t%0xA^xFIR_N*sPWIP}KG-D#Fn8Jv?@2TLJO*u~?pp7#Ss1FchP3UojcPMRpY=o5 z4b>^QoUMDF)4j*$2T!v?!br$29-qX0)V1*Zndz(J)it~r>FQE>*f*LYhypc#IiHx( zB4bg=szVo}whF(>T58Eylj2r3_Whgm?xszG?y;d55AyK*UJNqy;UDM`t5R61T{9h z6rHFvD~%)YMy{97L+QenYsASaz1STB=>ySpHlXF)%Yq&Ih6Iat^HmWkk7_JG)5de< zCD3J7fgYOC?WAc!1VeW3xwh={sRUb%>*ZuVi+z>WiMza8j0U^}>x|aE>&+OoHY@ zg)#S>G)|dyZi*BWM={pWMXySWTrXcq8WzfwTGtZS@|GR#m(T;%gp7r-yk#Ww&3KE89eLT?< zt35*zv-a0QS3deyRC$Bwwi{$l7!f4Va zZuNW^`mILV`s(0x?91>7CKxp8_(@Gk!l5zKi&L;4_W}RQ}VJdgImgOb+ZubM1>WV_S9*nZz+%Gd_DM9OM61$yBftXk;&a zK8C}e^)deJrIce0_3d`tt#;AEYP*;+`dVd^6PVeiJ&)Q`@Yqy4LL9Q=1V#z>A? z=gcwYt8%OobylR8%vRHX4t1#JT&+&{svPh6yBzho(IniD%vt3-MUU*J6{r|Mg_80%8%+|Hm3+a+U_CgcG4|!}eeyBc? z+RJvoP}MyntVtm*%mnCmbfsA2BeGpXcsG%)=ui=MEo?!_HCt^C~uk1{k=h*t0a=n$Y@VK0r54+Y)p+x)$x+*5>msHoo zSMq|OX++tnbnGVjE)J!055|2S$E0~4d$A*QxHw^PsIM=a;7ta?i)xcd`=oi^KI>=6 z9*&~uY(&r}Idaa?2{}W8Wqg@bmDA|1m$yl6h0nBm`e;vADm`@mXqk_msnjlz{nVuM zT(4KHwE0wuuX0;v-SV-BbA8(#$c_hlV>8`tD>xF7;M!2<4ebjJ=-%Pu_LY?=Sy4Hu z^v$JT=!gVBWPV0OlQ_Ayz#N3gTSX+dTA>!9S~4z)&#I!Rrl>@?AGT! zA!V0ON-Y&D-Kz=1gwN7e3W%fpl)Skrka!}*Bg$2QKF;!j8G3@(^*5xt$_|#f*5)%{ znvSRs_tAN$tlpTWX5Zvi%xNsHwcjRTpnp5c(#<1C5`2#D&d3{jMyhX+!Oc=XoQCjv zVruFThu4-J8uvk!2(K)mHwhZx@V4GFd)j9SS!TA;(V_9PVeKqG;P1cpnc7M{s!iJ<4MR9EF^@h7LI4lU!;Gi2Xb zOG~b$;%yHIlFOPb!J{N@I#lsCY&B&xH3uj1IMCRnf>+9SmAU|^rsj%muC-6>Q8oW*H&> z^mufwO(a3cf@qMz%aO^XtRLpG=NiKvXiPi1lZLC=kyI{m-4wf2N1SGZv}^kd7Id=S z8zzSlUac-!;ySi{@>O<=rL-S{MQnZ#h>0_PJ5E)PZA>RIam_rbCoeJ>phk z*gO^J7eGhk>FVxbTI%eJlZ%>VA${Z?h7c~y8@2QqdNdqeC!+Mt|JS)nJh@huqTuIr zk*Nv&StSJSfT&;auWT->@9Mimn*82o6w)8EKFMMy0(9@v0`(21*1xdS{?qEwB$e1z z%M%q;Pc&RCch_=g%=7h5t%W0^wOIA6)T7=%b3bl!yw1#p4(}kHBs5@MzbXEOjb*3$ zdbc7uj%s}(>g5;j)~jy|2LkX5DXlYC-#l!==GD13UJN;?bH50FNQt6Q9$U;ur91-e zjYgWWJo)y_@$%^z#GMY7yrK_Q%T2d)aB;fm7T;NmCp|C)`;!+w#2}&4GsKvZGCBRK zDFw*>r=$`vC_mf~>W`B<%ufFNfs7M``o@$*RX_s#E8c83?spi=kX)rMz{9Gq!D8H& z;M-~s-=@`15bfFde?XMI1IGVA$UGk1=t`l5c-Fc9z<8b zn+-Qi<0|v=v~JNKhZ%3u24XlhM&!WgNrOAdQ}hnmw#qzp7a9bAoahujLw{i-@Tr-QUHF?C`QY;# zkx$v{PHsekWbvF=Bv17$sF!B-)!9qXd^O{H7}K|uPq7qj<&R=LGi`UK6kYpiwnaVI z%z&_`Rtmrt0f3ofZAcgMH;y(AE0Q;LT_j^vz{M=g;Uo!-T$w|V<`#b~3B{0s@e@v3 zP22FEr+!o>YQopcF1?96XwW5&!DE@E*vdw$QETBC^qF_FZ^@X4YM={hA{_bMw9w~E^z zAi%;O^&1@!B^W7GT#PBam)6DVcJfW@+obg}Fy$I5eY2h%YTBzYrszv;cO?T)V5-Sc zcXD?Tqu6`8$N@h`-K5d>hQq*V|=ILR>Zy%M?S}X zxp+}pWld1>EOr#7NO#`O^pX8kr#aAIwSlQ?s_mD<>LDe)vrxUn-pB0jJDu+qs>wf< z-p3{)ojaS&_p_cX@rXU3^4OUSDMkG`^9xhPJQynjGCLP`8r>W+7+EgD%Ug z^SVe=QMt$cvH>T*SY&H8z9pO{93a=($;maTmRJR330|F$C%qJD0k^kL zp~}?tPeuRVNZZ8XLP>Y_swEyw>Q1fu^8#N}%uzqd_|JDnGQO~6Kc#j&OsN-^ba0)* zLI#KGPq0Af6@JXIWt^V<4w(4XnAU|lS^p|xT`Q6dYfE;M}g#%M`3(LNpmASD=e zY^B`$RXXj!c1cpvjHjSwy1&KzNv4fhTonD}?i@=G)0cIEfj=$*17bSeT@HY$NaK!9 z4rlXf%0gNoOL#X6ll_INbq|fh);enGwH(p^e}k17qGoheCrzsJPM9jA!Qfg%tWw0* zrZ7Ag-)$U|>1JH4Ih0y8h&F2ZfVZVk{aWfAoeYe}=Y5!bfBIImwKOzzxA;88jDay) zDBYM)pjdv`SD7>;`r9!A{Na%nM!Lr5T!Rn_(=qSUkZ>zV|o!m?u+r&hoijoXL&RsTy0@QlUw z5}SXq9c>bq53q}*D*m;{C8J8$y5E5o?)G^{Y+5u;fv$@0lD6!$ z)up{L8m~!dhW$06qlSjaSYS&fRoG*_ zoZlF3Lm*ULM8_It?b{6978#6oZG)2um}~7TWSx1F^>WwRKk$|xcvLDor%25D0*`jc zd-!T=IO#D~blGmMmzo3wU-`2mWsJeR4=f7)?$Y&?9Iz4j!6Lo_0m*|QF%IjIh8r`t zFTZTPby9!tV9w5E)-O*EWOJWtU-*;y5SBTWuGh;p)=jh6>a*$Boq@HIn5!}ER|c0! z)_;$m6H~4BG7}q~^9)`2P<+|MzFzCJoXkDTK3}hU1P&<0s0)6LJWTwTd0W+k_0d72 zX^0sClM|LnrI9VD?r&ypoC*F-!ha^nPFw1*Ew#hR+kYlFzeQKVRq;9iT)h!N%fOR> zYfQ;1Q{}zZcvLzNZx6Owqdx6AarC5*4p;ft%J5o|M3mWAf)Q(O))WL;$4}-sEh+47 zZ|cPNrV|l-W7*7HUuG3)E`#J z5&+vPHPq)#4pK6r#Uq)GEAf1OZoAbEv0L3G9b4cm|48!-*J#UCd@6Y^cde@=Dr&jf zp2y5)+w)fu`=JEe>Wo;cTDhovA$_$7)%&0VwUnbd@Gh>kKahO!RVq#p`b%xZU=?v{ z$c6_D$*e#%Vm$)=UDl+#R2F0!+|K1J08-b%(Ai>BQ?7o1LiT02l#t zcQuFhx(qa-tarZIiRBp-3j=oxiLfdC!Mo8u?G!w!>ophL)7m>?QCd%~+8*w8Nhd3d z<2Te`+ahY9S53<8vMWtihqrc?yVC<2Yt_eq))emVC83mREi5 zxxf08PTU9dhoSrwB5V)nhz-DY7$LiHmf^KY{INjH$vNQOlBeF*t#!&OU-5M* zE?id9ywcn6LagdfzzOaS^$~v1IC&xaC~pJpa|nhZpPV4uqGlPgAFouoJaHE_9V`H{ z5K)sD*859d^p|W5C#A|3x52pKr7c|N!ELne`n)+9okW+^$m$cm+A$}b^mJrdS;cA{ z_Pn%3h1Sy_x4*QJWX*Yodw1Bgc8hxJxO9?J<5Sf9*8|Ol-_L^+aleyw=_1~Xo=3(o z?nFVry(3Spgu3fh$urp3KE&JCJk$2*GA$;-Z~P8%DsjT=czq*@crTv1h)rE!?M*Z) zPyO(9o!bABeAcU4Ch?w&Ybkdl7uWuJowPAgGl~f4DrP~>!(J3c z+)Hlwbo+&->?T*8T1WUgm)^Cn5+U?S;P2`RevaVc1f0S=mZ!G7#zgoDau#u$(I_1F zes%BXx>$NRRbV3R7r{|pVV>Mow?CZpoOEb=@P2UMyTf5AEVA>aPRPhNXWYekY8ZKQ z$0L8k-W{qxPmr%JmBMB=`W?Yik`$%Obx`yy^R5x~lldBY4r*8=Y!RqyHb=GJ{k)8XN?Obj6<5VycqvrxQ$h}1ZR}EU;Ts((EX{0B%}GZ zwoL+=*|vNhkvQ71bfuU;0)}6q8qRptHcFRv!zlxhYlD!L zg3Rh=pYzLvvl|cF$GY~l$+T$X@=MTH7N}AvjX5HQni)MS*#zwM*?yg`>F-@@lcx)z zeQ2K0FQ}EQSe<7;lhsR1-3Vd*DKcOffuPI{**l#U ziFo*!`~_N)m0NrVx#v@M+T6JX88J8VkHe#J^OPR-`~hGj_qGOA*F&XY@x-46eR=r})* z7$ocQ>JBpYr^A5UR-k*K8H(;JX+&>ZsE$CzxEMrj6|9N)rbzna_+04|;UF%RN>|P*^-5Pn{Hb zhwS=1e~i#>HQ_s;Y@a8kP~El(ykxqa%Myz(k^(h@>8Frd27Nh)mPGUudR2%X60R6k zAB zylC9px2V1^({*eo^5vgeNUW$N%3EcR(7j^fT4n7iRxQ06<>wj4*8QxVPc=(2;9ME68JL9a`fF4l_<0+@{=IMJKuwQtZBn0Kzv4o_MDYj=gmUTa;DO&40d zo?5FXwCr`6dT+>_3k@h9@VbnV#X9;=fwfX(o?gVlNzcoAO?P`PEk7fR)yfMa@mx$a zn*&hMBE5xySz+5-SUecrTb}Te+45TJ@@$-MR28S#YV&5R-yRXkYx=P$E(aG?ux`?X zvkW_PBy7>hk~=i?Vy-Sh)+<=+I%$Ouf-{1aZZ{8!gi#zO70E$;x%Qg5sp^(g&ed6Q$%$^z84rzd_f#bzn5s`>fF7XS3b zs$fF-LCM*ul2N7h=gN3@5W`L}YX86p|5xJARLqDPsEI z6zc%HDj{T!7(RHu+iCicL6{f_T)TX1lWMHu_t6L2noc)uYdY3xt?+YTHj5EOp#QBP zBlra0D<_EZz^Lyu2=HDa78I7U1C1VgQz%s3bSU(?I=Y$hH0%YjLhv5-;X(}OaXmEo zon=hyS%#WI%A2FcBq%=q1ALx>i~4u*`AgxzOng2-je6p9EFoXUXOIQT#AjL$e0E_o zk5rpw@eTA}D%hOK_sWT1#^$9_Z0;Ld+>&eYR5mq-wy0l_`#*=zPl4=KiaNIG{~10n z?fCcc`G03R@p%FYinE3yVR_{UbVCz5~e3gES5|2oAULy>IOQ2&%U=eXfDq z4>UD}LvO1qi2CP-+{&Jrso40z)i24c_uhHH?U+3q<0|$0{sJ=HGh@1E0)Yk7Jt+rbCzR&W zhY5{UVtWNtaZ<5Etzj|6+A}m=CoE1Faj-`iY<4fAT<6H(D!+{)6!? zwf%SL?dKpYGQJ1B^yTsW+KbWgeS?t8o@1O@-d5^aq$j&KyXi9;ccgQ??g1fSK^qbI%iE+Hnp_-cH%_Q2-t zPyZ}FQ+k%AE+yrEgwGe?a{t}&IhV8%d@iFNJ@Gk$klyL<*)P%G8-Uz=fu_HeLVsuQ z-3$Fa3;q4WAG;9sk6EabKz7SiS@`Z`p(2%jjj*1TE+Rx%TBej|b|+()Lb%zbBb!cl z4(>+?-HcR$-fxhKZ!n>-FTLEHMx)KSaY1N@`g|ibzo_S$Y|gHYJ!V&D=GXMhTwO~B z(FXhn^T+=m^!NC#{(gYk^hDiQLd^cYsV6OJI;#77B&9}KC%$$Klo9(c*t`CM1l{djQ+{*~d)F)-(aqk~gS{&v zyr{h^mL0v>yS5_nIhVa_B_n`E?;Q57ai%%#Xw#Hd+q))E2f0$To4xCS*gOKY&1*U@ zoKn4Mgn$7OwRP1iI?__CRlP%1^KUwZdet|xDjo__3oIBASn$xxy{e1TE3m=B%7Wi> zx&O9rK-gUF--wmver@HNt9m?Sxumh32!~5XHy!$5TdzP;b!mC=pwQ~cHpB-d7sp!S5LuYDef%8x zhiW1K_W@8?01i~CX7dt*?;ujq*cFV#j$tqeyLeWZH$pVRbPb26Qbe@$fht9|Pg_&J z#M|}Y*VgD2|6iEzGfB{WzAx2tMJ&vj>oo8+t+(I-n_?J zep;;On+~n!xB&BbOg%H4a`sSzn__R(gPdhs?6YR{E>3Npr3o~)CduP4d%vMY1D zYbx1zQl^=K-Vjx)CO1bu$`eNJY1T3uV-w^fH;qw(TqF?{s*9L&bS`(f*PxAP0W>Z}vu_3aTc zMqLL5Y5H^ZGo7|na=S@j=5Dz$Bted=9aFiuN+stQ|46Y5RNwk#C-HS7+78kcC_{Jq z?l+H|?dHf?oE$mJ_nYBzym6ge z&g427lZ6(+)3@mmfwTQ1Lo-jrlzURjzA^SdY#S@G9NsywaEE&DIgYiscbEsSLSs(G zh(-w8%)26r+v_y>(wQnCP z*LhOP+@b+d5D{%6-!$er#VFg z;6fXZX?(0Jj1k|brb}%dRh&}vyeL^7o`jM`+yNTL7_^L+?`aiLVZNz5Iv+C4l8br* z{m~F?5!)~F_BCbsKb#{f7f}g~`?e?(OrH}NU@T`8VfUNQL})1(hw}CR^ca;)F6_)C zzEJ*!ULg=;#?+sq=m^!2szBQ`zl?gB%dHDkKO$s!>*E>Esyospd_OX5|I7wsdXazT zZn2({o8YQh6I5+Rr9K zz6KdJZLTV-7j^L2WR?Ci+RPL>k+n3&d(nwlk;PlND`qON99{)7`T7WGtOn)Uvqrzw zPhK2_^)EP^5Y@3objf`(MX|Q9nX`D!ChlDnQBNR@%NJKhM8VL8E^9tEd4% z9~$MVV0R{heM9`#w{n7xy0|J(`C0IL{NQeFESB#IYA5WN5dGk+T z*tNbQwj_D8^Zheh)pX(M#qf7ARl;WL2|*ghV6|Gi^gBMgv`gU|BYG0f&R`gp^gB`D zK%6RV?)GONLtW(CiDi(#|C78%ILO)ELERnWwx#tKz+%g=Sb?(5Y%fZ%Zo zG*}~;cB}k+QPm{mhnG5|M~g6;>pEKGBw0{8K#mp-Tq5JY1HRki4SU``AzP0X7J z%LKUP4hm?Kn=NKg(WohPhqfLL4yP$F)5*;EG+hf*naIV8W=(z!?x=<VyMsPUNc#sYKHu(<(t{Q=rOGxC`a+ z3`XBopf|0YXb^)??zT5iG{Bn6G^Aq+XweY0kJ)7E@?W0`hH8UnAy~z!`e0O^`rvWt zLnlNBRPM?*lDV9Tser9lY2Vf7m7mEf42PEuERLU_S+#mtDW{p=y*T@k6$91UXs%@V zJ2J#%tEg<;K`R$I45N;|%G_5r5RBlm`1ZwtrD6Zfw%9ZY@T$j8o+{=Fd#<53Y9But zE_wt+aVK5ve;D)pulhxnt+?lO@No#2OGN zzYE@!uQt9iNIq4NEd zq2<*>>x5@##v*&^pw;$_3zj?8^k~Y^w&M7C8M*e=eM>(r|3>(ptVdRUq~_8Dp=jzI zD#zS+t^F$l2OWH#`Xm~%ONVrD9w!>2eH$zi@9+Co-_1ZhRd(EFS#di`wkbQN% zB&uKg-s&_VS9lAamTapkj(>d7$+OlQ8KueA8yU*MFTQ{!c(3$xX5qHwhXL|c#uFa} zl>-okEU%?IeU&5YrX|QKjtxzq{JUV#I&FCP#>CJoCaZPv40B=gwNCF^&3$)l!H#ef zmr-$sqhu>1@O))~ctwo1KAO(5RUhux5G-Ufz}1oYj~56=eU-2J7d9B&;kw(mGtA!X z2cwB!)^D;{Qn`MRtbsA|xDQ^h-mZ z)zYupJ0P$4b~?J9!Nk`{$+EmkC==6M=%=|7H<{5mZ{A^D(-$6}9DH-@8KS$5GqCK& zTg4XN2beDih3%H^heM_37$fej43OIkI1Bi=tsb|}I|lFN4qvO~Qfh}M`hvn?d)4g% z?A>8&8@v}Twn(}q?A;yqHmK9Y*xCvQ@3na=i<1X0tS>+GA>7`u4;dUENUDWf8Hw8q zizgXVWLyp2j=jj65<#djMaCy_I`j_`TF*eYQ4w%EQxM5bf1dBEMaeSlF*mmIqG#um z43U6)Q=U368nT`c!@cQotGhv%;DK{@2i%RAe4Q~TwD$&%QZvNjSK;@#ea+>Dd&`$W z(*WhJ9}?@7JK7^D2N`6^MKp84XmOwU$9@?~P|y4%I=uAA>O9pjnu`F1G15G}x)0kB zF;_9q?Yfx-5mz@om@QTncSm`ZrL5{QA@V56ZI5zqP;Yk!*5F-tg(YB5sExPup^oz?gH7^+kyg4+E(&|~hZ5a& zPuT3)CMiFr7Tlywn)`L0{I=$rtQj|GjgTYX1>s#{hzF_nER*f4FkToyg?F)}!zwHa zI49V)%~p3Wphw#00|j&6kyj{Sj(~F-`x6cogm(dYIjkGi(HRh|!#L;09k2&4Kzc?s z6RDNa$7X52tu zs~u~Yc}pVxM?AYUaEh;`4L1@ur6zyKO&cHHm72g|TO32CLI6CIJ_xRTZ`B1&!&41s zYWWdHT3NZAE-#hy^*5y^`rfmIE|0{T;f-eaUP~xZ08Y{u;ZpCWrUj|V@{4TR<*1b0 z&OKJKe+UhK1&X6T+vKzVEAS_svEpw+>wU2^DJymPZXoF~*Ugo_Q!(CTvPg0~BGT?F zs#o;YUKQ130Qi2><%66^HshPgm5^H318hThy9WXK@*^VQ+lhltt7oEQV==SYzVh-V zu0n2EyJEZNBGYe*_-uJ0!7|S~XtUH)?dQ+9O6*uHWytFaPXPOltKwH?okvts82OE^ z*M;6aHbv)qPuo#f^${tXOWXG0GdHwbibuA1&%q_GGk@i|7kT1-p1hwS?4Hxe7(14@ zmf%D8wrwP8TH^ZfT@rHh+u_fE2@>`WVV@I*v~f){!oW47$b>|P-zNZ}-SWK%4cENa z5088%a#E-zAtK`ZK%$P;Y)1HKQSv9+CayUXvnj+)slL+=kE7xN7j5v4X&Rf(6A@w^ z;zr}T zKWm!tR}sPYMx9g!-d+hI_gL^G&=>ZlG#NMBlTV z+^a;6;wq>U8H1dsHEIfcio*g2hFC1qT@`PdkB802H+UfCBm8$syvb!GXHr}bJj1xa z^LUJ_qD)%xayxg?rnP(U^}}hhr>X+ZnUosHS&QDbaoYImoR`J`Uq$`2q;96#mUG4^@Mmcs0#En?b3iUOMizPm` z_X;Uk7OITj_?)=jA!;?mtau>e&t0LXfs#w6a~#nMIeM$mDzmsgWY(wm;tEZOzCiTt z9b&v$p?*O0J9mX#xoyu1^_XjcZlaS#4HyOu5(vt z4N%;3otcvm9Z0;rye6TyTFo)b97v7>doMHHgct5FgL(g5ILS3Qc%$($Cg_3)#P`X*% zd1Q88@5K!;A0`UnDpY#s#l!ln~Hz59Y?z+pt zKOhEQ7s6TxLRi3dE^!m=K46IMh4Dy&7WHVIU5lyIR-lYPDtNv~{mmTklz3U>16WERXaK z(E)cs#g77Uw0DTT8e-NlAda28S}y`+Kre~By}agl?|FXTEcJ8p{JeJv zu~8uX2_R1N4v}Nl>m(2-&s{HsK-8-}GY7d)d334HY41JD!x~`CDY87(JH*>2#AzT- z_YUz}4KeEs5NFO^t+hZIxgD7~9UiYa4Litg(3jZ<@2%kA3$ps6*Vr6l&k}pK*Vt6( z7+$e+kjM7UGI(JejH<=|A8X$NA60ejJ(-y#Lo&%3AYg!iQG!x|77b|P5KID@M3gWg zFiGU4El$&sdJ)b5D&*nGXeP&L?QQkyty&e^(pFpA>jP0BLm&@8d1y5eqTwO#@gRnW zNqCs~{%h}@z!`*7TrW~w zg;VHwibBCD1_kR=NV}>{v*v1%U}*z`^*Ry*EVwSs9fVV;BE?lW#V>e@Lcu8p1#1=u z>l*37YWw6fx)U2 zm4Z>^4#Fswh!j_06y#+Yv?UnDpkR&QV0Bw_yZhJ^7)M3-z;JyD1u$YsgdT)b>=r4m z!YMZL6orCQ3<}ow?$09|TV+@eOEWi?6V^FY8 zLV7?ggjjG7speyRU%BWAG$2T|9kb>hi-i2xz;Hdo%d0q!GKsYe0GBLMgxSjgDdzJO zg@VNl3i)IX)=6vb$w;sUkxzmbc|QyJWI(XmMT)DinRj`LLcwMR1?y>~Qf*&ZbH9oN zYY_RQTvRILlL5i9i4<31Gid@W!Da>ptM>wdb=sPHIufiw(-Qe1`2bU||rS{H0)P_Palm1^s< z=EBkTpD>FrktMHW+o}y6jl|e!6f?=s@`@)+0MI;6XS@*OcF@V0x zy64X##Z~xkMd}UCumT|BG*4#iOSc9y4ZV{DUW!+;DDXzj-PJheCf#55Hg4K*v zs_koQ?$?oE4YKa}JukB2D(jwyMT)ENl@bA#;46cIbu$O+j5YU6A8~=tR1C84F^F2P zvhX>2p7--r7|RZxqEIlFLE(B1sZ`sy*4%F+(KX1rXN9QrD(jv(B86D@3^;%%3b2Iu zGALMj4%P*0?uAIO23hy~`y9u`Rn|TKBT`(2ul$CmC=`5UP_P~l46 z5|(wv7~)qhoB|mL!k!zHvbdyZQYQqCdYH#SV`}6YQ?eGIJt66>hlE~sz!dvM3Za)B zFa=q`N^jj6*2^YdDI%%<0C}2GVHDtoV>@UsV_)f$3 zFAnDD{_yV{ou@fR_vF4D-N6GnIupLXZpzVpyEjL7JHA(b1n1ZIUW)IH`*U<{ALi(G zH0J1<_T=cy_|C=mM0`)hxBG(}-QQ5hZ+7SC{(?F_Lmk8M&A0E8VS7dfitNZU0Cfpn zt$?Na>U&}F!*3-@g0-I=uJo9032=x&m=q_d)ggQ0Qe4M?ZN712!KGF|%lpBM`#$Lk zVSY1F;A4bcVc(@T@LNS-Fs7`7)$$az0TB3mOz>t=1h<2*UDk_ZA+UAKGr9D&d6#RK zeAk%Q3zs1y*iY9Hk>xb$ZbG@WOD;F&eTU6!=3CfR@n>Qg5 z!Osf$L4q{cZjH#ugKA_g?-veZ-~qhDW70IY#Eyv?c9WKYB|O6O>Q^X15a~=?;w0z zKmR_)amCuN77dK?sD6z9ZUBrQ8UW+l5jHr+j{*eR@E#E3oooJ|FkTXY?GE%H&Rz6j z`_Tw&|HuCl+sg(<_l^*{Q~S~VbO_zTQe6DfX}w}Ja6a~wP3ed07N?oJcS8M$+ZO8L zU^0HoUU`+mqU2jdsQW908CMM}Q7`$An{ams9u~vr+L&c97n2jiIkJ(%eRfd;rTs#U zOW(UA*xOrSxS*T~rtepF8mocVsA{=GopTxGkH1BeN5KaMUd?#bYt75xcym#MFSmCY z{eH)PFXI;=gRK4^?Az5KH*7bIU9!Q0|q8+ z65YetXm{8QgI!L;@wr?TcfS1VRbT{%yy9U&@hQ5!tW290TxD_vVW@y1Wud{;30sJ- z_z4~ug`l8=a=4%^=-En`x7YX54Nyx_*W4Z&MIl3rK8yZ)i2}%?A3$P+tGeoppRN97 zhkJsu&SHSMPIgjOYV>bM$z!svqeoKK_4G){(q%svCC9*}`%WXA!b9-iSN`-7_=o@J zbQmh*qBz1C`vb;3+1^Ptc^FozV=+u^>hx_Y%@x!c@iTdp81@Hl>|GZ9@6H{-VD)%euh)tSEu z2h>o&^6Jdrh66}XsTEge`oaOU?$L6qGk+Zppdz)Kt26&I96)A0aBr0fmAI${d3Bmp zo%sd@!1M5OfTatrS?)3g50!Z1KZYA*J5U+5LW_QIW1%yuNKn%^vM92o5oAk7wY%mmcG~`W}&2E z6ZaB847>JB%7!rXuuiR50VA97C9HWa03~HLKL=eVmR5 z;;|j5X|k0Q5|uw~Aufsb>C;gkw}G~mtNVdyQy8x^DS??Cy#)!#dKX5vwiVzFrk4qw zcn_{w!Pd<|Y&ZvY@OA7(K+^yK1o{%FSKy138Tkvl2iRgeP$w0{eufMf;pKOHhVee< z$2#3cLR}d-X+7=;%<(UdE{zjb2T-ngMedd`=qtIGwUZ)U-aDAcWsh8$69>CJ+o8QeD|4 z>;)p^F!++^cfga%qv?i+OK?)^>RqJf!N9M>dO`B-rs!MfaK<9*Ik}`x2WRI+n@sLR zM{q9WxV%(32u9h%}5*BwN0?5Re6_dytmgsdx&_(SAr`>5S0IzT%S zXcGA{{74BJBp*Kx)=h9u=luhJb2EeTu*`9CqO{#+G*2817a)j`fmiqvKH*KAfaAd+I=$P(OU%~d={zy6Swr!r z@8RP8Nnu=QqeW^Y@Q&k`0HDAa)wso5YRL~|p&cN?kX05-hrLAu$aB+FbSKS|MMX;1 z4D>SW=2-?tCeER7hh;l99fnu;!zIPFV_!Q0${#Uf(>(y>D zCZ`j~#jsb!$^iDNmvP9e8fxF*9jo(b>c0?No#jh<&U$dwt@GLMF_eW#BOGyxoIc|p z>^KjdN}2O0bAdN26?JS(!4ED=F<@wfOcc4KkOU*4WV%SDL>Z6srIscVoRJj*jSO#r zoUG01EpQiEHS>D+P`(KKUA;|h-h4g6wCOlMKwzeK;V{>=%7$djH$EF2uwRdxk~_78 z^!oHenR&^$z0jMNd@!KJ--+6djj@aK0cVT*OkE5B%*mXa>^7nDj2T1ZX*lwFle8U8 zVi&1NL3ubKhPT%iH1MYeyLZ`uub{yeAW2*lszafHDfk4;z&HvrDAxnCfS6pfg()Sm z0A~5>^b@$T4c9@FGXP><&!DX{vgD|I2kXLhwze{I?~3btHD7*|3n$~sz!PEgnl%x< z3RWDH_lP2jfhl>WgZYuEYGziMuQW=&o^F<&hz#CX&b6~0I5L(j7yu@M0;S1lAPkek zRRY7oe(Z!WIf34F@D80`-xQqkUOd?0~B((A3}-K`{@#APcz z*Q`22mkz1M-A53_W!&mhBy02=-E$Jgg+qjIGV+yb9!7jIYJLt+0z;~i@P>d|3IHex zAyA|ml0&O|2_D=9huHFUFW7YDNtmR|w*c7iJqqcENw;z{CBVv8SN;{^ik%zJenx}( z00#B;OK2rZ9#j4r3UKfQ?B)xcy--%FM+B}zU^bwdv1Ymm_&EYxhVmoG2g^4ZC9D1^ zRKdZXNu7=&b}uglM>QqSW-8@G7{cuYp@S{!>8phV-5bg+sZqM}P6CV_ z5HM*auJeMI@{=`p(kwF@mr3lcGtR56Nibn%cV?!{lZWI~UpS8zullms=P@F*cF85U zQl@x2bKDQ6Puxsg6-9m?JVO^41~^T#PTV79!S$lf%oul1=-F9} zkvFN^ez4c(fLH6_2s$K)rlH2$?94p{yrQ)X{xWZ(AqsnT%uL}_pjJ3t&4N`gL@dE@|y_I;qQe3OdPJN$bZedn3FvS9=4*^dm~JcfG$a83{!EAOn! z>MS#$K^aob*NF8Qvwhh{_3kH(KI0Q=vNx|2Yo15dkC}p*^`B9=Mim3DwgM?D%28}+ zO+8V`nn?4!d~G>BS=vK1_HdF(PUJ@`f+Hvxcj1vvbFP^=X`0T18d={3*HlO?<-F2stwAeUdgi*%&ewG&eyI(>j8sKOI98=_sX}1O4z#<12D)^ zshIEKWis>uYDpL`x3CoY&O0BP`TNipO~Dc9`*s<==XR_vktk)G6scp(}2`&`Ysh=vxD*allJk=>x!koD>$3j1;^g}^8+4N(l zpL^+NrO#W1_T1$@fNAkI^;5rQlqJmozXFJj^4)2<%iX0MljIaceTCD=FyBU~aSlR{ z8|0YUsGPgp7fTm4bRSk4ER#N*uYRZ;6XA2*7yVj)4wQ3GfbdAPHGv;=%jx#C5yVAC&K3;pNHWo1qb$o`3&??)c{JHdoZ>meO9oOIIen5?R9BQ{K5f z@+_1zq4SQQlvNM+IP&xEIVEc)Jw{~NV+=Yu5rKHbBG90NZ{;io@Ra>gUjCxR zgaS64k6l0A6`S7TbDt*;rN@bz7~QG-Ty!|~G1&G(n1$ffbDxY;{yDNp zB8m;C+7OE|GL`4OGGZJm)2!FKNaWsfdl7Ga8 zRbpt+z10uv)l0(d0d#Q|h>b(o_24?Lk($MqA74Ap+PpRi*T*2_`eoh>cFqFT)Cr)J zn1LQQVuBZ`4St3jYXL3yJL7&B;!eHWjTz0_EF0jb-37P*zNPx&`K+FDl5-`ne-kXT z1J5C#$~whuY>b&=Q#at8I3TW#flS-moYrAnS8I#%+4Qdaah~@bFm;l`heI6toxiuG z>S#xme`T$yG1~woMPMFEO0Taz4%}45H$V$KXuyVWu`d6F- zQ5tScuaEcRjH>%aN^H|BjRrN>sK!meUYahx*$pSj;ItZiA@uYDk6LV_O@A{b`oIR) z1hlGagwG#|4hB|{W=_@B>1v};1TO$=d+iYAcAfP2PNI|hlXUMuIHM(v7x#w*D~f^2 z`Nr{iN#hSf)DU+JOFLTFFTR5qHMPuX@JnykD?OQO^ajj0V7;c=7+j2`vrmDTx*7Z4C88DBOdsbazpI=B89IMut50U02+ukue-dukIwB;X_$uCJN`z!V}uww>k1wIJOy0t^tE^O~%bfu^ZE$KmZCr<9d(T`F;1a|^sr2UCv z`A8$FBJep;^M(|ioC>e+5Sk8QW7H16L77v4TSJ4L9RuS*Nl?J#VcmO9$(Z7fX*5jn zt&Z{>su)pci{gaR89|iLj%VbhQ1+dMPK;5yi#Qfn9W@&A%;hs)ghvbqYXOIW-U+ki z8egG4b3>ABXpG5I7bdH@=BkXT@)S_(oQGq&Kc$XxEnj5)aJ@<0`CRuU9mZ;0u!Jb=9+*3MCA1OHzYowP_He`r&u5@Yrfm-8)a3r95d;_C z+!GM1k{r<5n{g1{f$hAHio}uqZG43Q_JS>qdlu7G01Q`a;gGy03+HrkY!-r%OFIGA zKh`s}->iP1Z1{*UI;j~rJ`5bw$#u9snguv<%N~#PhoTJo4Sx7T`j&vFV%0fsEAei#YpVvG=> zB(dLPWVuEz@+PwiD3JrtffOLqLaC9lBv~Gfk|DqkR~IIM9B$|2;CD6|r%t6cIgx?j zsM(O?DlNFlgprcA3}eYwrs)lx7{PNW^c93+$_M@Qvn|eA02jE(4OCZ*+SC2HX6kOW z_N>>dJj_9@6Zo;PMwO8*! zuO+iOsD2^2uL$`Rw6G^d_$m=zEIJQ5kciF$O;EQ%^M|1Uw4mRLh|rtt?D)AW`U@i$ zj$^35sIz_!!4uQwQnAci#I`Mr=&Y$fAianN90-w}4QpZ1fZ^?JOyLBXtU=J|3nvzas5>ul zTJ3B8_62>-7n{?h`|jJE#vPe%Gx!_ZkKI3#0ycEk#i8mE>J#CK@AhUnHkSh3-`44K zp@f=-TdwYx<7ebixE=tWaj0kKRNX#Rz8K17s?*;d<&K(D7n5tNr4(IDi7_k#XTSo7 z%ml>XA%Md2k&wCIu!WYEMRa+FV9D|*?!{s@P)3twUXM45X4)L1Hk4mx^iDVemuP^? zMU-sfYX1JFip^0oJAs{03{Gi#jGeu36xiyyUo+caBw}ZKFm65dW$rUPvI>#VgRFBK zFnz(mp)T=?aRmiiVqG&di)i4vr5Smbzu!DCx3<$qlF5T(&Lh#-d05gwsv&2 zXwnSf9tw0(w%uo(iJjJ6Uvbi2x(s(#m&1P#ZcT=A3r?!oot#M#8`oofPXE{}%)0yF z!7BLH0;~qa1=rrk=^a;1E4>p}om@(_i5onyRr)1*Ug?RJ9wTCv-i~qoWSML6BB!-M zj&|ZM-gVO3Hhtfh3tvY1x({*kVxKKWdfVj08nno1-78z1*8MUZYUo9vTT4IW%(TeU z5kk)?p|q(3rlk}y&P==BO;{0ojxdk`s372n9O#}5=E6fzhms0XFtJ}_YNrnFWb+V3 ztd;I8(bANR>$!)$G3+*k6G@rjFd~C3LwejSge=~uWx}aX^{I8B{(csKdB=nvib#)MkTmSR@ovg=5;8!27-rI?qVpsZ5^-+`Ap zl}GiryT?jz8z8k&J0CU5|F*M{$Ogf$OkwwG(d->Shk4QKsl|Bpd47jqdkL<4*Hu9~ z;(3w=F)oVpoJL2L-%Ztew>^bGb`;E&SFZRx6d>*$@pV>nF(#$vgVxos0Fa9c9By+%)yw<6VB zk*2Noc}@UXsDz@Szd8j>5oRwyNDRfz|czw{$Bv}qcA`dy^hIYvYiO5LE(Zae50)URzQor%rh&8!?|oHqC6<1;>S8s*i?Wdi|ZqWAxhM z)FyADx6q=^2z$BQ&3(6 zM^tetktNVq@Bj_KE2afZ7#k@aXNA?LCW4vtGb>Dq!4eomfEC`1t8kkkArJx(ZkR-Oma{RxFu?{X!6WN1Ov0_xDQvms}HY)vGZY! z1*g+%ACe7Af*s#q*$qY4?+b4RwV-y$c@3^>ST}ZLs5e~EW$*t8cLi?0#;B`z;C8Il z#5&j_pUVY$bvA=z!mn`CxhQAU6(&wPGB$c zxa$$89fN)kJAm?V?^0uz-y(0oIeohFE{5)*vzh6N@ufzkS&zHJ|6)b~?wigSAs!#j1ugXN2*Qg{=Vx%4Q9SquSLIOf=tnsV@#EC z&0cYF3A0sVy}uJEp3h%f{xDXuQSR}OKT5r60J+D!Gr}qV-Kg>w{ zm(#U}XxZy|mH)oMe~FM8gY74i3MpOXY{v-d+Ggx$%cGPF zy>gOzkWg~$TK};agSd!Xi{8pnlABg3d8&~&BC(&YFYk%=AC37w{?e-8LzI?0Pg6t4 z?97I*I=Gd<03~pmaoahbUJng0uV*uUYoGpx|8Bu=4RV~O;`sR_Zn?OxpflQ#k^`;rO@8G3f8Xn5Af?Csul5ElLTdd14ZC zp?gz&gvcOpeFYW;jyZNiSSP#&W*{qSR#>WRQ*)I8AD%}OcWQTbyn}KqH)5My?X6<)y13KKf(S%lQG=mR6qL5yO*aocDF`$e&9`n zKH#G?aRiP30E}KZSp=R&4567C75EiCsOU+eXddXP7hsBtB=1V9K;R&-oEJ-3BZ}oE z&EzHBDd$yPESEDx2~`(YKsj=8t>l@E*y_DxaubLJE~_$DxXsxJAhj!bxKs980;l;i zHE~>^9nuxL#p|2db*{W|D@+STqaM?QXJU8lMq8xK;+`}P=`XK!mnL=Yp0#P@ezgt z%~ctgTPQTh=ZoN*Tkyf&L4MUouwu@si^{3>P_UPW)L>vONBltfsu7MHfy?9>1AW!A zT>ACT%n6y3zrp;lbCgwO7Vspucc^CAL7Uxg^rhX4`jHb+LNeCB)y z{aLA8grjaaXS=PBK)V|(R9a0jUozk}>*-bCoJ}031ghGI1(cM2=uTP!;bK-jboA#Y zH6H8<%s>u>p+hzUC^U7Y!eXVG8&M6lIgMh&qd-(5PnF)z$JKp0q!;o(WYi;QsGZ%w zmrA71y0*zf+>ff(hS%j^t_{u5l#T)gx&pgNanwJ|kTG3dX~ddmva|z8APSa~o}09@ zzafCQ4wL%s1C=m+JsRs^9XRVz7I*+5O3zB^*T+$jRI?wySZPa7mhtePOTXeb zmiPQZUas^!C{Iv&{zLL)A+Cux0yXMt4nMpIm`D3WmuPeQP@q;`W@90H5$o229#l@5 zT@9&N$$DcoY>D+ZJn#r1Z*R6JL=+FUOYN${1Z&0<15&E_6(Ce4rO6~%0B&}zyd3yD z0^LKaF83z;W) z0DcU2rBr{3X6fKIdmE+KK{qA_GfX7cMWih&tJ+3W%g@~@5Vax_$yX+tIYshJ1XDpz zpnbN`=%9n`qAAosA_HwvWF%CS^T+BJtI?|P&$ z7BzZ40uWPcJ^#j|iO<{Ua9v<2mC$6wU9NnlqOKj<)sQX6NIUW`Yl%ernu!A9fG{`( zRWEAM>b%QpFNLL8dn=RI8?7I#9)@j)-tNx@@B()9_fzOpz1k~=GW^v=!%j;}QjT|T z>8Cd9feJkw2>Roj+%euaX~y%p=-2+0|W*cgg~O&_OMc-Ji{VjpAWtN5pSU#LNH zFBh8Ci^_$}iWr0MH3W$YYZD7q&XTmvDxXAsdEfyyAp-z9v|qjzR1;k_6WaO0Ag=(* zUhO)VV1cF3D8nKo_D*ck$kEwgA>U4F6uFgcG~B_?9tYkD6Vp*1UgaSI&r`I~T@6v; zuSpv2P}a*sMI1OPwGyd8nly(_+D2?b<)M>$x*E*xaaVBi5?3nI16Yv)!eH>tO~~Nm zGE9+}CBT!Ye<9Z+A%yL&29rD%7dI2ZMNB5pKeM+{Gf9-&h@qfRs$AZH+hQJ8KPV8$ zl-`a~{Cc}92DUO@A0Zj#zr@SGg)gAN^W1l2@*w%P5IB+bAYbrJkwHdvD(@nIrSig% z`5A2@l+i*A)5Z_fAP2jh1ByJ<08h)h@pKXO4^i_a5Q+APWGWZt(5WNr?14K-iY*iD zAskIJXr1MI(w--|3mxFE+V(tR`TYrTs_$96^F1tD4YAK=A_df`EOQnuKt`fcHU>jXTjH@K*SOh>o~;(8z(%;WRNs>eBZfZ98{Z0B(E3JLiKq} zUzp^TJ7CHP-_l=(WPYSgMmmcc16#+geAI4HDslH|9Q1V%_oJe;LgfY>8QXXcS7c)g z#10tlD#39&J0oYqv?;R79K;2`8Bh`{GjDPahtJGA7`JHTah}7Kdli@sMq?+<8^hNc z%7nC&`7rVUtS`TWEv)otvfYtm`BHpI!;;_xWA!Y;$jh(x#Uf5N1`jN02#&BIk~j3_ zO#?-n`l4Tph^DKS!v&c8Qf%%^A>%ITkOG|7!tZ7dj_ARPn5&BcO%ZH8YB1nET@O!q zISDOD?-G#XqHsCEL1*8B2b7*i+)*|!Pl%nZuBF)ZBBFqm?=A@a!4XNQT%DfESK8$r z2r3DFEQ%>VMPC~UBYi>Vc@UwYFG%+XN<~fY3o`Q{YHnZ9L>@%#>kG=@L2pC`-OYox zL;1YkwH)Kpk0wcn|V-kWKc5?+7}tr!GjJ&2A$_YA4LRhY{f^p>7_{I zq|w*LBN3^}XY-(r$e??9(22;PDjvilgMP<@PDKW7=0Tm2LA!WRPh`+XJm`F6&gD2Bq>KQ)JLI9%POTn$3eOkwK+AhzKxT{KjN_l$TR@IH3P!5a8hs&+F7*-d+Q;> z&}C$8BD)Q&P`FYbeHu)HJo%TXy^}7~Mb#RmoEjKH&vBny22b_zwIf*^CB9xe2)px7 zFwD~Hv30&if_ZOMCYVBY4&9)BKFrxn%8YMo8^Jx?!yOD3bzJRYat&pN;E%T~IE_aJ zN2~yds1(MNlvwR8R&j}se2s!zt>mFl?(af;F zf=f<;qFY5mrO{N~%cGCMAQMM=GU(h%(kIBAMJG}q*s)taL?gzrT0DfI@??Ui4Cmj{ z4`-%)M^LyIyOLQVBw%sZio}Ly(L4dAI#`UB3nQ~p;HYIdJP^*1T+Gj6IG7!}QSe#s zG+vMfHPb&ckTiU#2kUp@QmpagV6Xw@SP3uT#_#BXDvmd3Bz9wFDox3}(BQ4?A{?a9 zoY2?2w)FaCIFy4<0GAI=+f7ghN30BYH!;A;FyzK24GtS=KP77dXx2-hb0gCJK#N_R_loRlqAhn1KC7I${82<6M** zn(Kd}1%unsgO%DM{;)Q#8OEmc$4iLUvegae`3yS@#oOl>u{!`X$tIoJgy34EeZIY@ zJPP7@@a+^J(T5{Tth^kU_S*SI%Pb=R*b3c@iw2YFI9ZZ9gt=ef0 zVi1y;gB^qzugxUXeKdG+egw7HS*#{{L8_s%j3w-^@GZs_oo-`#P?Oky^KVjiI&o|) z7@Y(vKs8(~x6SkLGim(t$RW-`w_%4uaSqMYu&?rHYh7mu{WlaV6YjrVXB!& z)2=|GcWE+It<{BQJ(RnkYJb6~BAPpa<9Xj+0tsBV;j+uH)a&lBPG61# zG8>BdF&x#n8AzH9<)I>5>}(1xen~58#JQytN^jEoABB^lZQyRi1b9er+Gcj|bfo5A zO4??fO?xFgxL_m#T^(})h6a^`KpK301i?{~-o>{$!4}zNP%_!zVLRD9Av4+zl&l%+ zV9#TaRX37}Rhe}<4>VvK12**T-6~9~oktw!&JVw;D4pxdX zM1pWh-D|?7e1Or>5Z1A^?gX6rhGafT&@8@({c4oBo!e zG)rV@+b#^&yd8a|fti2`MzHI^vqEFVz(8PRny+TV(7J!&#Mlv1SXLZT=+-oX~2xH7dzOJtAs)rD;-5Kfp5NKv6SX%jJQ zKn4zW54JDM8d$P`Ft0BcyYWljHkN_bh@EuI0wpX51D%tjou3#~f>G`vUEDe-${nu+ z_3o6e?AcNBNNjbCL{e82c105?Pnoq*PAOqg^bV?h9#ji-#)*wD@C`iuF0RKwRh)N1wb(EHo4T;oA*bROUigD+Osw=4&N8|&%!p+W zzrcV6ouZSTrA|sYK%JzX^SWs`I(twmrYt&=*kR40(J1-e0;)noZ3{-^Jv1WSbGl(M zRs{x;9MRG=F7(@eVqy*Aqg20KfHs~3#T$S@%#h>&SpP@H!uW)68I7$(69 zmgHS(@{aQ^O;HLh!7v$ccF@Qc?I(7?QSd&sfI1qt5u_iwZXs^0A~uX6{VawwnYTvS zs%ECj7janaK6%QFRr2*S*2>qgVc`RzY1Md7kBQ+5C_h%yTIzzsL%D3U{S)9HZ?r zb_S~VS0O@^`AtCOeMP2fKULTQ*{-le*;*h^B3MQ#EJpG~V}^t#q#lHj30+V>(-o zbV_dZm%$yFAaBA)YHsy@9y2zSQbJ191Mep|a@YzZ6P)CWZxtBveA`QNG%j(FYF;2% zLfCd09ObV<2xgyP%oAI#mprvd>Pm%uaVb(|#K@O$`c3kzLMZ=oMVD_v4hQ=X9FJ$g z$ZU*8<*e)$z!EZ?r9(Fn z|7bEPj0;coiSo z=H4&f;4+al%HG}$< zdPenwM}0Gc#0tc=AU+s8iJ=N4xpec({{*xYvARW3q|u?Nctx6Neo;_POJZ+Q&jFB~ zCelPeM`FTUI)W{^xo^z9H~3IHOZ>gj>~Xv*9oLrmJ7NstP826RQq~~;xMENG!(w%PxvBUd9Q+_hZbE*`aXO!NG%JE8C5hog4nWqZ6_ct7B27}J+q_D`@iO)Y z5-fL_k?U4&EoNt9C-V>>dkE|N2^FB3k$m-Vc8tERCljyTpY!4ju92lQlVN7sLNn82 zFeA89e0UUDAUNMN_11SM6+&zo-niM-K|zXM5|>~Kn8oV zuYkb3SUQ&~)lS9J8cm{=q@>{;D*aYS70U5z{l{bYjgd#xBraam%^q#sgfp|f7~Rzv zFu}?-*RU8-a5^uT5E3|pXz~m5A)cG`l4mcTxFiWZ{3#f~bs5?r6qKf~KhRT~HypEB zfjqLn9j`WaAAlQ!!^+8CvRcXO)Ru>Wnh%~i;;{EE3PJK4kURy+lU;_049i?`Wk_Cock&Gc|2^&0O*j`)CO@PK zyO6Hgql{JZ&wP0u_TaQp!4b*O+raP{ve(^!Oz*b2u#`M?@HDiy=N-5Y6&HAsbQqkL z92}8?w7?zSvLRX8**zG5Ww^=}7zu0vV}KdE&x{jJr?d>@4&{WIy-Qa_Me#D8a?e+&bry-jF4lzbgdb|Ce{2BZ;;x|I4)O} z8Rs^djha5~QZ3Q}1ju=~$<>?zuNPLoq$Gkpx)LC$*Tv2lE!7-HzufMZhu}0ue&Bh0 zsCg#}tUaq!Z~(q4BhH=RiYrx`^tCa8zag?VQSGs5>o_j*?XDg{e8O&pQUV1Q_P|Xvq;5hX0Wy@##@#YT!&z-H5ko)q709=d zX9u($;mSq*`k|pn5juf~8el*vvWU!{;+6hF?mwI6y za7k?nJP2Y_=={;cs=HHddJbC2o$3b%dwfax){Cnbx)Pf5EbvUATZRoTCz!mPPBz!^ zPHzk>&#~jBjfK-NIEWdEcl4MuGYY0^38nUBA_WdnHPSv!c$`$-oopn7byU-%eAydV zj_wDo!(UyeEAAd==1}zV02JTKDZYa!ewm>75LjxD^mZ)4CDq)32&dIg+w3tK?qKbJ z_b-tCL|$%*o(fI`gxI-7g1{GOlhd0>jcC#bev2aRb$fiQQEeF>a`4c?Sv*E!5r?V<+_WmoN-h4xb@TH8upJ=$}9n zNJ=!>6}{Bjh&l21vDTl6e)Tucp zTG!w>C426Muw1qt3S+LFFIL-1puD;WW;N+E9H%g*+n}W%JI{SiOP+M5&N!}?4#3DU z^GI_~SIp{bVHUbj%aYzw?8qrm{cv5S zS68jW1cx1ae8i~#szL)JzE{!g+80@SMI`o`SEVefuIM@PKDh`}XF`$`Nn@1?++v)d zR?zK!;h;v#UG78bV>By7lY~cpvvNXWzBq_M$G#q~p*reiXO?L1#W_f^SiN_CI(fHu*TUE~|s^ zF(&g_Y9#@B77}9cEsA1wNLW_piYfKI_!7S1=+jq`Qe7!fI981X{5aL7b4zx%jEYZZ zr91}&N2L-t9SM^Tq%l((FjFfcIuIoeoZ}-(t*b(G#P^)_5g&CNMSY3U?Q_uW4~M%wllL)|6RSMTyZvF&?IOs} zyS*j%WL%9^6dSr8@#taYg(R50aLM94Ql?IVv?Sc05bK<9Io zG`#%YAIU+^%Ui_b4e{8_AIK>-KwLUDI~!n3SV78t>hzvo_pqjD`lMQ5f)B}2o2>MF zcV%!I1&_e_IhaG{b@pe#(Z?$`FG3p^{OC^!*C#-xUY(3{kfX+dmp%$my^oq`G5jv= zg}g%=2j8RSzy0lRJIFuYq)TF)NaqY$^1z^XH5!tcWzcXyZ`t52n(ANsAD z`4KFihk8RCf`r6Sj{zHT)i;CXbcLKwYX`MP8 zBd~`?;K%QP*>qvBF#F#T(-Rtk^H$0!nQK?b*QjgDLxb-EwlZk&1(2fw-Vo0p#WzeY zkOJAQ(@*RJ_nU@s6Y@eCLxV(i1R9p5kCGqB$t2`|-VUG#yBj$>6`WqUj2~2k! z8Azj${2HK~iaN0xXRo~u7+bwdI@k;_Np*EOi6Dn)yn+QPA6R-%RwCY1+JupVoCDE684d*pj_EQzBGrrN8ml= zPBX5E`A1W!c^%%;asj;oA6?liv6_aqQP&ks91JUfWB5Qb_ID6$K+-JZ#swI#P|j^E zrKdt9wy~UEKOl#M7os;m1Kd^j z(lFZq5*tSpQ~P**OFp*CzTilP!4_aQxrg(wEYS;gW+dR@GX>oc0{#lr-wfp%VD?bQ zHfZB<1j0S8`jW_8%$i0an0+ypBr&dD(c^r76Z`b-u$Ch=a&QB+(7m%;v3()7&=R4H z!3Vt52L1wgv3ITRhw1&?6PeF4z$4W#U@S+epNgO?8vCnGR$pRRY)CbY_^Q4{3lFH> z@h-jHfc=+LunXc7n-D?{u3=3;9E5@%Ll9?VK3g)BTk`=qogsmK>#%n_A%0fo61HYu z)rQl^d0M`%t{@60d~p@uL3k?M1+W5k zJq8%u2zei;@~}an8N3y6gyyRt#u#`Hi8Hf^MboC%i%83te^3hr{(2>FZzfYY<_Gdl zOr;feRS%7kyc4KaZI-9k<$#09lccwX5|6v4`XY7u>fv=ybRd~&_Q+9KH0DHE53KnD z+zxgc?hml?&R6Q;O8kTpG_KlGjD|Pa=n#yK{l|p}kU_S-LNg}yr%s`$WlmM^o3D3z z;kZXX59gQJoq!pgwG5ntpYX4}20bYbDJ#N*Hf2K7!5?@wfB26y+#t$?J)8H`F1gT{cM*Enxb&9VB|VLK7j!u&h5Irm?}|mjSt0`L z>jB!2zw6j-5qbk~x0<8RBVPr;UEnv%+1PYawxx zhp;gSTE`-H;VOh{BYj5N)rZjqdkI5|JGyflmqEWxju(&v`)IdeK?ea64KiX+^LY3i z*7`lcAa{UGids65PUSxX${`x=LTa1ui7FKB%$zEZw3ol}SDVhI4<&mHiXQA;R5wwF zI5TgPN0yXtiA;bs3?(3qtvEED7^)4Ic$0oT{c!mnWW|}LQ6m9196;|f<7l)K)xu3o z09~MS{kRNEUiTFEu5o52%M)CZ-S-65=fY0n&p4cDljLb87`;#6I&Ph|%e%x}{u%&r zu*+bTLYjyoi-60{-v3J&)8k!}OP$tMc!tFsg2Y(D z(`4d?p^HA4XAYE*{4KmAY7!~dA+v(;g;Ts^Q0nrY0bVfIjT7SIHP{*+UIMJBJ?H+Y zq+%7Jl4WRUd$;;jClJuj9kEK|{yngR4Sc6~JxLHryxT1JWqbaFdAs%vG~MeX8*^Xn z7XF^f-^nep*EbpObj%y`E&GD`xw^C^@M~ zp81G$Dl=!ff8xxv>D@n4-y{giH=%Pds2yxFPs(@dZbVXu$v8fMeS>Em?3al0dQyGj%BTvKt$2(OGs@XNN^Cya#YR^0t4rb1r$vX1}GZ6dI;e@ z50(tDwSMc*IvpL0=3hWcG24a^vBg&d%FqR75_r5iax9H=vqYXG4L~j9Axz_3xkZ=i zV2|RAA;%-^?0EzLHU~$LwC&~JIMNPmZ6h4zmpIA^yby4A@;dp**pJW>=xqT>dVlCU zJQVEYP3ki?sz2rrh~X_wKkS6--5XWj2atKkX4~OSTOd2|r!Z*4IJIi4RS)$7od7ne zoWRhRLtVz;n}pXAwh1~#U`u_0dQeaSgd$MDO}r#v8N|^LcmW+#%vK>R+fK|*K=&w5 zgH2L*bbz`p%)EB&7DEK*I^nV>bBU$mdbnk?_$rOT+d@2Q8D;_AiSBrt)oodSoUanZ z2&3b<`F_B5Q>ZVc_!7a$$j5%nAU-9~_qC0_@7xYBYR z{7*-f)7W8$4-;9Dgm;7#mgqk6)jjM#E1)3is6*0E8k) zohOK@Xp5d)9m!+MN&kUl?3=Gbb6Q+e^h!+mR3vKn5K0}Y0R_;~6w`nQz?HCNogC?1 zwv~GKjF7-io%UmO z!x7cpfl8aHSq!45Rc5PkV+aKl&r5)y_Ilx_@>f!M5ty4Hu?=UanQ+4HEPyH))Fm0+W5Onvu( z>)_$-X81W#>dm|^6d0V^SCtP{QRJ@FtS3?7`sWmQ>~YUg=cPdP0iDKBq401I39e0H z-z5SGYJM_!hy`Nu)IthQW+y2ae05ohZx);fCVS^uy!lB>g~ukyzdYA~1->?LflrIz zX(&^aHZR3HFG+hny`_l$v1fLQfl_^HA4ulP`GM~i0=3?4(=apv>9v?`EDnea3j{$1 zeuSr?UY*^P6rj6x$r(j9k`nQOSYMby*|67l2?5~scf&y?f2+{Xl03!Y)woRZkdK7& zXZf`JU!Z%hCj%J;A1BSW+C1rlgBQHM*$9NX00m-@KBcCfW{ zhHLCYBixLy+g`y4vG&MAgB(Vs!CZZb`l;eW^-Zb|GcTtL2Rr&Dl}c?^c@w-Is)5f6 zW?plmeU(iK(xI(M6o(CqOvr^2Yq$jEq7C^aUxAS_UrL{wIRz4iU`L{IgkgN)Q!PmK z`lj-iv=Epdpl2!IX&#?%wCY8HQqB9QO7-zz5%~h;RCqxqHA=nH=pC)Tz=0Jl<^|<= z?=h+zu_2g++Y25CZNvI!R1dGuN8Ly+kUTq34^~}ksaFtYcU6`rgD;R9HK~J;gmstL zf>#^gAwJ{0?#Vn#nXU9xuKK#17RbTrX6Qdc1+W`7ph2R7U1qXb$tEvV^T3ZE6Tu-E zc!N}S&eIc<_xsyo9E(k4>nAla05Ahd`Jj zys&LNg23>_Pw&@8?gk_Q@02`gVRK+!4eDI~^AGJAsOBJv5o!AX+gWJ8ElL4aP6 zzgI+vdieTH2e48CXE21tQtfB&k#H2p%5II)ZLESUjn!?)=1eGzl@pz9E$u03?Tgb) zG?;ZQ4Nhs;>@;{-9+qW4ZiCC-VI%J4&jmeaB}<1>T76)$Q`%jR7{^7t>@$e#80w=Y z^rDl`N@X)V^`HY}oXlSg^-gKxG|z#*+jLvMsl=YhK_y^`-&1`7wN**VVbn@;tZr*1 zT#3RD@gApi$xqdo&PgYccUZzE)Q`G9N8l$npvopBTQGs3`+n6ao{qODZP{f0LRFVe z^K@<{a2yk;9cQJ|PUPN)@;*d)A6|>{8W1}yTGX=mDBiO+^Rm$#g_;HFgcqr6;AJOy z=!Ht&ZP`mN3wEEC7BiH55v8^wsULsue&=tKN9 zB4{jCkHC*9@FS!@iL_04N12PyI;EuGgiatYi+&{K8tBHWkJ0;-&O0!#@7(%QHZXf& z%E}o!UHk#)eK>AUm@ ztD}I{(5!S+Do3lT{Z&YW)E%e}MU{@t<_$0V1XYILgLrQ+)QkV9>e*~mh3E7tk#;|j ze-PE~LA7O{pcTQbRX_tuL+GxHXh|t=?@K)SgDpt4AkB>5CUp6kDahF}8^vWkg?Oa> z$|;q#;kg@VuE%>PQg+~p_tIWGLHK1D7Aw1{Iux-|qnC5|J5ITE%=@1%i)4CkhSjz4_o94Hba1^u}CV3Je1h_b#p?37O9c@AZ-0cC%(Q{4m4xyV+1 zQP9jr6vf9&RuC0Qo+mNz`k(`P-Z|e_>VgCT?COK4k4hgl_5g>zzjR*GJ5lhWY-p}N zN`|(|Ma!zC7{u{Z>R3BZFL+1b1~4Q9uIFrJIsI0Eb*%m*6)!z%^91m?$}EnA$Nn2o zNl$!6Ll5nL;@@})JdQBVBGA+ITRLZPdaXC0#4A`7`ePRgJcg1#qLMpMf;=Cn4l%9; z(Vn%9%!8U@j|+%J>OZuK!5ZdBXN{-X)w{adLi@dwaD z7-6eB0n4xi;yrjSNHd<5mhRa)206I5XP$xj-191e+D;R}gpgO#0@#nBJJA83w7^9$ z5>#Ihowp6|;1>G~z7plc^bm-aKF?tWM~HLA_=ptfPr zL}*(_hpM@q5^f!ful_N%cETz+eAcZU2fpGN2N7l>3pQg+gdN!gi< zhf(>;M;$HGm3Bf?H@>!IaX2DFstV5`O=`Pa$9v7toj8n3f(q1PaBd zKeq=ELZ~Q(WUyc_p5SXGU5Gn@xMP?cF2l%=T%MeN-JMq_`^y(@LmlCk} z+m8?uNHD8WdAsWc*=yq0pbeD9OQ904Qc@%2)&7#U;Ohq^ z5s@-JeEGy&kG9j4?C|q^ff_1-U;~ZHBPemJ7~|0$cn0oAGzHQJK&cNb-~lU-&IVPS z>>pJF3Pg{6(*@BRWmm0At!@#d^dQNaol^z^lk<_?9~p$CRs9vwE^6=_(s%apGY`y_ zgMqv7<9js?CF??ury4yg5!f5|wcFa((sl-3+(p1GU$=erSqtNAkbUOw^4y1HQL?d( zeRig6XSQ44_Fdf9vc9b&@Gp!aX2TGPhqccwU*irPS>N?<6n0emM_>~o12e%p+gsaP z4^PpZu<1kv`i8czk>6@u!1SYnfrn`P)1XSTNM3ofrH6j9S9oiS|t$_sO#e;w={XPg$ZYd1u6-5~X zpP-H3Z-#+@`$srKLtA~@(Zf^50qD=#zTdK^5qY`4wz{VYqirp%NBhFsj<$7D5tm!r ze*byLi|)~F^+ytJL17ez0ud%B@KY`gqypAyJ!CKQ*-C zc9h`w{}0u)wTOD!z6e#1fY*7&UmW=E!0x);sJN~p0*V7C4xEVF^+Q;gA4tP;vbBMc z=TfbUI+f-yda#DW+i($4@bK&O+vWKMezS4w1s@f$A+m}(5-k&9wzUkD7%q5dOvU25 z3H}oGnSa1SQ~%$y3$nWxbyjVovUQc=vg2Anfiw<7T}tcaT^-nSCx$22c`%(bPg9RUR*~s7nc&jHGhY_Xg<@RtizhUJ+H@Ey}?wemkn*dG_~~83N6Wr z6ptLm$n9wB1TXNneb?UB0O`^Q=@P?S*;DDn%p23TA}yu?J5V?^9Fk&Nm9$5{M?z+eAA%bf{~(I-E<^j9|E);?a^<1FX5}SJ(Nqwlk|KVpd^$a z+NCemhQN?UGqz)ABxD?cQXt)SR>BE++fgu|&S2);B=_x^Zi{<;8*4pCzR#l{k=olp zsnHV;=Ia6spi($h3(JRO$<5NQD=&!r13kKk#<#W* zC)V9dki%Vyv;uekLO1cOWo4}g)n;C`dw73qXor3@CXbP!(jG~o5EC`F0PvvjcHFdr z+dim<_EFsEnQN{l!pab??DH;Xf@8pJ#uA*q9{Mfe$4za9$JL(cxyYZi^-qA_VcjYH z`g!nY9Ld8k?Bb+f8~EpW#JcqNOxh{c(E1yPhMMqDShQ605AjVVsr*}^Z@ExmLnP05 z%nwbtSOGyzc}L!;fL~DD%k8$b<7sYR($Nk}q1N^#tUbowzN9mFkhX_C0o2Go?CHgp zARHrKvRVF$e+75uKPE${B7&!51iuoN`7dLrtLcBuG!}Mh^H**sIfR!Z5t+hW%lj4mi|zt$bbUSSj%VLO@86`m4x#ONo$ceemyajPa&=GP?>YRv zg1^7v?``~9@z;RAQTR*1-+ug^#os0T8PRmkg^C^9F)P77V!8#Ia`w`ED%nJKL+aeb zEnKtncFXu?KH8ktX7>X2`y#O=QZa>YT^vofE{;c*Vx9z7+IHUsdfsKx9&!}3B2gS} zt^XOZv*?UUILe18-0CNVFxZ$Y-euD6VChFuE%+L&iFa|XM+|M4qrI)Ty_K1HA@EwE zjf0axcYq3j6wp_Ih3N~Z$`oN3)31Mq{m?=>-B-SDQ7(4IO>E1xQMx4|`G+~B1E>UolV~Q#QF^cR z+FN_8S8eV6`mpr@(of9;11Qyi)ItFbiuQ~{F)HREnE8Hd@0pN@+V=l_eOj8K0Ds3LV<^Kz5*7v9b74rv48)eBP0>r(O; zIDH9v*)Cu3$4o{yoE!~MNJ`+_&H(qyH>E6EP*&w$;$Gb9uthpV5O=dkI+WhWT;W1r z5L%=&!xscdkZ2_7a;HsI12h#4MXYM{%S$rs^PQ!&DjO7ucA;TfDs)@{h&O6{3LDDo+KZU zE|;;`Q-Ol_W%%TL2?xkgUy*?uEQB2-(F){5Y}R^0e&uIzA?=zhahPV6FPNO?I@HM? zdzx98)nbk7fwK@7m9|48Yrx~A#;-4iLo~U_K7BsJK>Mlpxzg`&#>>a%oJi^^z~G%L zzpMxqjg;-lO!u1n1>$2EVK|Qi9fRLIC)YY=7rs%~qfPnfRD}5-y@DSktr9#LRZH9a z7c)NCpGsNLAp`foy)tmdNOdCL@P>H+p$0_N?z&5IpUUYC&Ug6x9IMtKOLBymu3)jlUC_0*(`YrfvN5H-y z`bC-pD3!v49unFuz3hX?_Z)B|o^hZAk#gsXE&+(&6ZWlR9gJ-l)ChhDiHL7KZ518$O$59 zEtmQ$>c|rO*?PWUrM4~bjO_i9eoJGs{uGenY!_%)M;x&5YTYs{1nQko1!7ofk!PgT z$TPCaTAPOE@P1!pPaoJyk08@bCHyN$Sv&Qig_;qX8d}24&y{e#lyIGt&=EM7GWPi< z=4J7$RglX$Ga5;H*elIsJG5wR-5hY&;&L&NKNZOMV1SUlbSW z*XDJr4h61UV09b-M-E7yL*B1prrzp!maAy(SNhy?0gvvs^-B_}qJ7&US6WBHloz;C zExA&g9396i=mpiUC%%k*n2^9s{G8ns%WKq6C1I0F(wG#wi5Goo`P-js@d9w@(e(^6 z#v`XR^3;1f8FKkjN6e%3@+03Ky_X+wDr<%7H{jHz0hwF-5qauBM+5}`F++6#0ZM;= z^2RcpGqro--8n7Zi}-W$w=}24S7L3c1>nei9l(mgXpz1C9;mXj=$KUXi2Q_?M%GqW zSPO_o^h-%io3?O;+3vRt?>DiN;+;_99c6W4k~}JPw(q}?zId84q>+!vPxyS^q}`9m zPdJ4)Y5ODcgXdD(immFS+t>JWo&__s#RSoFElkRv@HNH1m~enW(u~k2CMEg0YW>ZU?evXlBF+9K3Wf zN<5whL+~rD-%D|A)}sT-t%irdYAi+k_-^2Zu1lG8bPg{=t!K=TdNM4}K*rFs(qF7+ z{D<^c8Q(Jerd0RU2DnZ#vHh90u&2V>@F>hAusL8HgEF0M7KIe+VqoFVr9$LGAu{uqLQG&pBy;xc0&0ACT$gKE79jEF}#baXnR2-%tidGly^3y@hez7{&^{uLL0>R zdy|Vzm+Ay&IM_KMTE*9B=c3yG4U!>$ecV*#S2UuCuc8_ked389PrSHx4}S+6 z?kGpBRxt9;F&m_vrfO48s~v;NkjuLdQKcvTx?qDexD0cj0^by6jVD0?RoaZtR%vqh zVg#6;k%(CGcR9btlD`2&qFvk_&9GpBCn2DEEe*mHg{}ot-2$YjYj)Lg0r3_JKwV#f zae)Q_``?q%F^UeU8k9W{3dwN|jW-p}nn6Kq`-d71`3iN6d*{iZJumR{ep?q9RA0a( zv5RrCz361>R}}wu2FL%rjB1>5Zx3}v8l;xFrW)PQ-Fya}1*hYEc{rfMC0#*KMpX~L z@jxpYPJt~7vDBDzTc*LGQD7kRw-;DX{h49X1&i*74`IlBg%{q7#PC8uLCy*eO*Mb6 zZjq@<*T64WvlO&8SstXeH6(frtN5uY&B+q|de%FuI<(sXs1{!zGA`P;#eAURs ziqP;02qjYPzZO(YSX^)l19Akd$?pvi@e}U~VGgwuZN3a<7!Am;+n}k&-Rp@vGwuD4 z3u3aZR`6-@q++6Di$pEO@WpEIDIe@X@2sZadBG{4%dfBxEwVg+TRf;#6DZ)9@TO1u z1suC{w+B z#q7vbfT6Rfp65!>@lo^X6A;!_EqUF+;6vyHRaC54#Ljs^Fep{zwBSNyFGfFF$||Ck zRg?u^O)*qok+UVC^w>s^l>jjB9vj|Mw@8n#L#ss!wzotH(hx#Ch!~-(6f7izJt)~U z8A9fia6!w)--XYQPAwIK9Ti7D0ce7(WX;ay~TM7mJv4-k+9Qhu+i&D0Y%;JyR;p#C%L7%tM(@&Mrm) z9Qhop?N`u2br@15e*@mMmbpX&fs@gdR#Okf???$73#^s@BWEwsI{I@UY461xWuv-X zM;vODUTAas68)WT;j0EkKiSEaIwSKO>xP?Xsk;gTap7SLtUTFGvn#<8 z0x32f7bTQ4TOjV6^7gR-5-V*f=3fKB44@kfuID$*zq`!5`_$XHYqa5)3+Mp|v#9 zA8Eq8GisneyImUo%u4mUTz-S1>LqqJPZ}`&pKUI)es8CtH2vypgAFRlGxBRbBM`!E z>_?sfg2B5jV9D5B{*j-qq9~Wi$mPcl{ynj>hM2B4p~pe8wwb4VmXf)c}|@jFE*+ zsrhMLLVq)2?t(RD*=bgH0wGaKVonqE_g-t<|6`_)_s9O;OYkPuJJ)-!zjq!UCwkEm zkz+*v&d^#~)l?jZ->nJ3`Dhkj;k&@>izxjz|KoyaHvi*-X|_-)DRIALsAQM>lFYu) zG>KZywpN}$@U9bYTNg`sol$?GJga5&{>P=`w#%t!9RVCS zXJEIi+?(9qAV7{xzskogu>_fR`cTTyVbkc-}ikicl6!q9S?JeNR4b|*1ti; zH3GRc28AK1c2!ui81bL=BvN&#tpgtz8_#uw(nM>;WtFpJuj%ksne+cEeyv{ z3mY8I^6LZ7OLs!QU?s}y_Me(8vhmjtGq*7#oZvr|x(5N(qGIb?s_nX! zZrufZ6{?iNzwig8;UF~B?@{J1o5N^NJZ8&0*mQiQTT66ZY?*j(9FvI32mw~D^*rYC zl$P)ijl?pocOKQtJVY0Ci;As3ppI@Hkh*k3tjnavuxU0p$T!Ffb;Dxbr?nXkogng{ zcHnP|h*)vom*Nx|H4ikYEV|f6$B2K>zF+l$7^2H#)ECe`)%(Z$`y=SOs~!)k(!1-x2-Xt6q5-^sBn*4C{=59#_hsZMD`1^r6@3OxBf)7yXQ2FzETDl!_x-o;t1G3vji zXn=sb-H~|1Kj>#muk=$jrhHIiE>Vpcz~{-neBH-?Jg_Uz6c|r|8u4<^YNpRQzCkc2 zoIcQhGQxP(bt-sI`RxA*yx&ynNvex0VI#%rD^~RlLbdkOF?i1pDC`eBznm5_o4q4+ zrgF;-Gm7OVHXfOj7oChT2 zrUqFJHO&`NDR0H0)zATE;mFx%6z}xBZBm4+zIg*AO_rfJ6K@5QUutApszo&z>i`_! zn~5*C9y~4`h^AHaLaW%>yyt2Pr5t_fIc?M*fw1VD9C*%@Xe9&Eq2x}^SpLZOae^6H z?K{VRYS?;kG}X9Ij%fZ&*eZoc9_nl$_Mw~ZIBNjMB}WIpHZ zYS?Sk&n8h6s7Axr)r)ix^iSqR2CVUCnM7FEeqVDGCYlY>Hr>#|fO-k2&4Tqk1P9%X z4deM7R3v1lQm95C(8j4uXi8+ImxBobXww*ZSdYELv{K6{V^gCwuxkQx(1|XxKb9j^ zi{Dy_<~S{q+xB{qYZ3o%pYcsaoG()qhoG0SB`APSy$w~?L)^b>pm!t4Wd_su)!V<2 zn5jN%WQ*%&F7-K!+ZBv3s2waiEG($+?N|HOx|vIih5#MIRKA?r@D0TJPtx>Hr!5xicAcdE6C%&e*Fyw}7Ke?&x4bYEh>+BSG@bL)LU zNSQKF)qbwF6jfW4RE1X1#cKz*Mkt0+S#C{2qDtmkH4XRVF4mMSH0kWt|8MjPqpcaB zfL^RS2_*a$tQr;QjjN3{ab2j2O7%vsF$0vBNsqmAJ+DJwm-CmJ2Dpz?(^k!fA(ZTX zOEpgAEa<{`5<8Cpy!5ao&?JmGZPYL3weEQ}2{lG|G2())flvt_+Sz~<%2UxdQSDRX z^!bsU8XZ}5R-3(!C*#7C^?V(=kNN{g+yM}1$Z^vo1wNm}FEn61;FHbp44yKKH~RZl zdcUXEiv$jGrlM(z7uzQT9l6by&LK&kojPTCwQrwr5jQAyv#&k{54?Pb5OU*-+_j{ZcoUF!d@6j0{fW)`)LX?0o(!% zg8eU|uuoI4e^5c|3JP!Ow1@L=YLMXYaIVUEF`_qez+2h!xI{b3t$Vv-z2IV8rHxKC z4L5&thrMk8P?Bc=P|LmuP)jN9^MLx6Odkb+F9*~uFwh18N;M=3BCk_mB31i8nA`K# z1(6AAM_|+oga&3&7CQ_*i}b9~eTibScZ{CCQM_UZl6gf}S= z7dlKdhMPkBc}{EbC30}}#T9>1hi}0KgzO}filZC2f;L}`g7uhcy<%qSY}QqzbaNIe zMgl4Ob0o}{TsL!~Zx)W2Z}k;4CCQw-2}p)7)n{+9h%ZO(;$s3OAuXdT!pTEO%q0Ek z`AKvo4&y1$TtmRB`4bsTb zcc{y&9v;yv#{Z93Zks3p7L{DNB{u@&f9}d{<3PPup)TO7uiPq3if*&gNXwPmtTVYM zAci3~j91V@;WZ)2C3msdcAVA(q3U&ukvGFK2680t4BuXEj-Af65q7tbT98|<%E;@? z!!%em$YKObyB3IVSBdi(nrm2Q{rWW^i{f=HmO#f!9^8aQe7&Ty{=g@Fbta6u^SNAq zPkDoXFEM&!Qnx(U@foJP@D?Fb51xz_n`KlAen8vhM)g3zcQ|_Mn_=Zr8UVGbRjKOs z_j#y@-2VQIXYBU(M?zWj?eDR(Z+|l!a{Ifo(pr0WY)p!d4P41(aQ&MhEI{QYpwhaL zvSc^jW&SmsZoLDSA$lG97(v=%zA0^7ayWMJ%K_hJz8=nzi{B;KWjRo3J;OeX0a~py zSU*2g#-+BduEoa+o{Kzl7(|ykKeX0^LOJ zsAa$gkH;9&yTh-4DX@FZMOcJT;KrMZfP2r4OoC?RPtDlUOj)T`FEdh)T=?qf0Hj*) zs7L{Pcv?eb-NhQ>t1nd0#`BT7WsSm;i*~1g+IODy>w#?6eYH}P^)MB-&90j!9(m&N z$m7H#&$-jr`Wij)i}ACQ*FD~sk~~&?_k?Q@qzH$x8*roe*s(Ibi~KmUnd!H#^o}g? zULr3U%B#&h2&-7<@qG>7Jdj*lI0w0{^XG1QC-NdB>Ox*p=i&vXkS`fV=!4-n-KxCisqx%z%6VLF6|B2{%ur!vNc`HltV58Toi>r32wf_*BD$ds zOXI#Mu)!5W29CP$V?rZH0w=n{y5=tojEu8lOf$ZioxNE+z1)c1l=MP9jOXBqri|A) z>ecHo`@?y9rulmlVdP$~>XxFdOcmq|fmfc`WE@8vp8HZLLA+GKOJ@3F>pUgk3@2L` z4P+kh47Na|3Qxl$$7wGn_e8d9L@#9stsG{3_zJ|;79NW%8lqAuWmfn(PJbA+?6Ur_ zQyn#;?(aO4R&)p|ZMs3|DXyrN925CGCw>SNMvb#pQ4|#F=RBgf>6`9U6e?6054O*9 zMEt`d<|3EbI&tGn5iY}Kgm<^u`hu<5W}DECmHBq8%!3He>@3;=tIQTyva7qb&SR^R zi|C>~G%r|$9B9#g{thrNNEks!^Y*iWHh4XlVXLN*c@T-P`qt}o0bA0oa3cu%PpG}vEhEg~QUvgla2m~r=YR#-cTa<}nP z+rc{!@g`MTzf+Og%-2tDXF_~IYa6Sbh-wysuwS#*?T$?`>NBSZE+Zc4+aGzu`Z}Mu z#B7x#W05nE8(2p!tMvPthKg_?=3FuRRLUlT9fQxsoXu!Y;?9VS+9hg|YJuG2n1898 zX7f5`W5Sr!2Xo$N!t(m%VtL&@&-XFAkmN?g3hyH0Md6naw0ni=tF{@Ho`Uaix|%xL zdkXnPvwt3Rp^bFd&Cx0W;cE$or%!A37EZs&I~wcl*h!vj_FF$ARWwPC@}K3+Btfc5 z;C)M6M)*F!Kb3d5`sjNhT1!rUyWOVtmHje#7pn^-S!|VsNRHAX?4Tl9=~<~U4TsgB zZY!R>y_$%=QPxGOY9%G(SO-zO7FU@0Yn`m+zLW*l6y#Dx{xy7MdDD5uv#)p0nRcGT zJ9bvndE7;eYI$33yOKNd7o~7-^TS9uxuH93m$!KH!&&u*e6K7XVp}@q7&9{Q0E34t zSFg~A?;=siO(rsErJWE6Uj{eVia?xWH9%1G@T^!X2okRLk6^KYG^rHcH}gx!c(F2? zHgO{)S<{D?u^k!ljTu<6Azu~Zg2m#T20X^dq#TgSN+%A?uv0A7Ysb(M#Bnq5B&fVm zOytvf<_NQeX%r&^1Cyv9U#-VC30})GEd*Z^OP6)@qbRXVkS$IQR5DXFNbe$OoEn~< zTVrJN&Q>9tnPfXVyYMIn@ zKEPTyftZ}SsKj-6o08ZCM`D)wCa%$fn1V3T-Hj9nsoUWs+a{G zNGK?EtvbpN&0BR2m08QE1Hrqws%_ykruFdDJ1{lhpSu<>hcHcM&UPQyVi97)znPj`Xh^ zgK6I~>zE`_eoa9KOLj$b!%u)s5IvOvm!=igzo~NCVygw$2I9%|VcHb+WIC#mCJj{| z6eFVu+hflMt&NY34UQ}ej_+V|J`*!PQzj_J=Bj7q-%+DZ&nAp6IeO&Etj`A2$6bBT z*)o;Zdnlf_mThQU+9tG4wwhw^$*Bo~`D)T9ine8>|sGp<*O$E3;pdg+84 zBQ>H=UvZqcx1@IBu?JI)0iXqkCe{I>7AvWTH`gg3BgTe4jz@d^b!5%`EdEC9HPtj1i>!m%q=N2&l-HJz&x1^5_fXpz{c(I5kSe{9w=}{ILLo0=B7wAe-sEHl~w3rBgK~4HU~x?^5CHi8h6TVqhnGvW$`% zI5$b$UHmmuvy3N$T!rZ)H+^)20#zqz%}baga=%L86Pi^m#(4HhfbLXdmZ185{`&d* zK)N!teUneqK7X&n@Qco$HmI<0Ms!k4oT^%sohq%Ry3?`NQWhZ19;;it^7I@=_ZUsU z1kuV5KtRkX@WzOU>7h}n533Hk!H3rNILi5q4-EavX7Y)pB5cs;6kvc@ZdiGk57kF$ z?I99is7RnJy>Y+8M*)2`__Z{w093^kb4zJ&%aC-O87ZkiqOcoP^=JMtDkTGZ)(D-( zw8jiE$YI?bG=5W!6N*sE9*MiF~bH*>#wsjESh;4#q9u z{Fk_v_l+JbuH5=5-HlNGsPsAo%TWo9>bX$h%bc!i{et{hB8nzn!ShQJRWcK*+P8N> z1438R$H59t|CqkI(u<{4V>8uL)7K2bVDrC7Uta}cf)4+PzSgBkjZykqr%Cx>`YOgc zK2KknwWDV2N>fdpyp1O)_Eq$?StU{QwNJrS(^pDnidG3Bgla1zV2ZAOA`s;7i|OiB zhF`o7NZ+V7x@!DIOsaInF?chcRP9>b!PF6$jhZO1Buq6zcQwr={5Qf-tS_RhLkOw> zh@=pa?6GyaBLjq$Mh+pUQCM^muL!EZC`M2P2&^b${u~AJs}$O;X?&-X@INA~QF)Y8V#eJKrH?Gc-Aw0la2syhvWlbznF!r2qM)YGkvs|YP}vrY(fhR{ucQj~5g zk|~t3N=(aO@?`Mrn%u7_W4z&?f7alj z-B>x3L2`+aG@)$GyhG)x1wH=G6Y7?ou6wrf2+gV%utVBco{~074tc&ygcx00D#=rA zYF=eejshVe1_Y{klioA?-93sXl3PF`L>bd=*>CWU@f>$A;exWj^=Rli9DxF?&Hs?)qvyxK{yp{bT3{oJyX z(0_*L_9!B`K`hR6WI}yg|1XK|{a;RWXZ$ln_YW9yWe~rV=#EN_(zJ8|8z~bS75HRv zSpNR)F+n3QsOgZyb%Ekfysit{DDB{MHzkrmrJz^{o=B zldpGmgYcjQHYeTuWwXC@^Orq2m#>7`MpjME929m93fl6NaZqO)u3QZ3fCbq)%53<@_73O_k0{P3Xg=0V{)!v9R`fFHD@ z5)yf7`U+aPdOk4;%LwtDT&SOH(5u+5u6}(I_5kVZh66!IA>SvxwCz5XkObB2Jlsb} zIJSyROj>44+P?64+r)ecX&&Ua#F$#V?5J7Vz3n~bfEaI_>b8$z%LGdS3F>QHH${uH)ZyP4CWL%hfK zNX*YoJH*m(`A-i%U(MH_`e$sz`aHJotjew2`zw>gc1v!&7w^Wx&yg*6kc#J`yTV7# z${ovmf%Ozhs5*08-Q!}`_%o|qm!V4-oH@!l`3*x9{+ALjEywygr76T%U z(dG90XB_`C4ff4HvPC-5r1W)%H+#>Y9`P=o-tV1BIFB2JK*K8O{U6ATSda9zM$FY@ z#LU$gJ5>TXe}{tn9Q=)f2X=EBzDcQ`TUO;|x`3vuMgWU4Ys7yE zuv4yIdZef+G?M&h#xgn_7t%0>BQ^lhK|J)d+>yYxVgQ7=?>JdrV-}65F%KvJ0SMoP zHJe_EVE1T=c#_6%knz}od^V>uW#wobiQ%hHg(Qm12iNZY-s)wN~}G|tuOe|x5^ z7iD}5CS(HNmbVk^ERSx=sBVKIrZ(y_*usoj=OlA8+2oM-Hi;}Yl_budlB)}s{R9?2 zC0=crIPPS#TFM4}R8-G9**yF!safYWlH`)14Vh%fP=b+`o^WkQPO^+1cXg_a7+{v&)9OT^i! z4`-@XaY~(jra!5lW1H1eY&nXF-|hDoD(lairw_LNoUQge!c~zm%;ubGE3574BR$lnM!kr zlhbWeR;uuR!mE3}K?bp8NyZ9xXX%ouWmD@0h}EV~*P|7?_B@J`6FVPh!r(2R|M;k% z9$cS=>?B5ZYc{=Xv)8~Xz_6Ivv9~ve=G``>OjY&PM+q=Bz;|J)X(P!WZ)%N3G0#?? zyn>P)vXJxiV%diE`S9*H*)w0Y0cQ{2 zwQ==jvYC+WeBJ7EXktR3v1GP{EG+UBP^%ix{*xW z=|!Tu2e9p*;Wb3@j44utg?ziLP=n-jsr`UarD%#u?7QVh6#Ujx1qmS6ElV znT%EkT#C&J*Ki!XP+Nk^S3%W>Q`l96uvj~X1<40_;sF@^);12XD zn+_aV@}%z8IMuCex)qK;azMJ)D3?5XSLOCyLf0-nLKV@jQNic>*2M*1v~Sx84)kp? zI}tV z>vH|+R7jjybhn`k^`CT_oVb1wZm`E`86-hb-u;}7oA?PW-i0%4Y;e3+&cK+V&3isd z!r$Y{=HQ9L=V9C~VJha|FnxFDHQ|zhSb3kMKZP$J9DB!w(bx-z#A4A(Oy-57&x4&u z&hx+5AKE7^Yl}DPck3qNmHkCRSPkfIb9v*6DdR}o!4qP-4?977v{5^; zumtWioC!7*L+~YfWYNikCuY2g^lEKVUgXW)#o@{~59aOD=I-KVn?;{KXGfOS--&}b zR}EVjvKO?}em!TGS#+|y=p>T`jk~zwy;gToZybgty-rz0>Im(u}k#dI`#uD+`5+{?4W$o$>?hI@l9lN ze55Eva0lWVofgaq94E(FTfsaK2e&U1|EE8KppjferfIuo)>uj=r`xUX8g+R@9Be*P zly{ z#^_5(j9Bf)ci2^kJOb6q7Xe&zpu|ZSb6s4scel8_$-KSDFg6dXa&))EEV~+K90V1jRbKTjOwAS?a{MysNv_Ar35^ z{*n}DcdIiNc67J8V&VAi)=YeL7deMj(X`k!o1&-LMS7a0FwMR$#W?Ml8lk;${?2~? zu6Q&}k@!)spCz}2yCg~GX(@#fW3pyqvYS~d*)LMx-a#}^r~jLOmm{1UeF^(_Iq}OP zm$YduM-%d#n~rv3Tl+E@k6~4r{@P5NcajvaER#FmjDs?f4pt!x!q;W|noGv`mo4Ly z^Pv4;RatnNlshX^5zP3&tW2>8DS3_zSa8WLML9tw9dpdNK1a&ONH2#^EwOty<^vZ7)~-%UYbJ$q7(?OEP@zgsg|4f z0H)(&R&7O^5VB(yWk%R5(P$+svA+qhh>EihC+BX~TsP)zmxeEh z<+46uA&|*BgKtNQGD$1qgLH;t8}tU#sg+_yaM+Q9D0lyk##hvsz7fXO_Fu_<%eNjX zj2*9@7NnzaoC@{xym|0Np`x0q%63Bp7P^Z@xP0U6h=-&!?7KbDzdMeQecybE_gYe9 zV3G5=41 zz_ICXT!q^(-wp5@u6_4)pXu$#z=P8n+@a7l{$oGjx9XG&79oPj<-c=JQ?FT$j z0Bl0ykewZP>PrHC>1>;HRKz!ocQF);kWuM~?Y#h^X2Q!g1uy#?9(y>XFV=V)*;AAv z*YdyxB?ylr{HmC%!=hTDNH1LWE!idlahY6NTQZ{WDgu65718L=MIH?II^m z0L5USHW%nk-ZXVAw9RYBK@fly!)yMDDu@3D=Zlg)Dvb~x?(ck&6;Gn9R4P*aKT>=8 z(A&m~7+=#vyx^w?YsEh+mG|t$&2G(v($h7`*cfaHuu$BRGB_!)gTTglhaZ|;MiUP$ zasE{3tqT^1jsb!{%$`lc!TmC49kasCeWx@J^%2$wVh$+ z{E%(Q0i6TGpE)qR^e2f)tz)Omb8M%6PTZ`YQyx~&g`6QOI3-nel3|B!ViyQP>W7zI zta5Ezd5Jt1c5#-1`{?EhyIL*|I2(zxl5BESwR4}!#@!!^+tl|N*^PUCOJg>L1dHrk z*Ie=;jp}YWI^5p!oQp52tzxNvjvVDub9GIee6rIERajl5b5|^@jXR7= zI5f^5!Tn>+Beo2A0_HC+|Yh|ybtS!{vx0*K1P`+*u5Ej001qW`3=cE;ww(^(GETa6CS?u z3;3f>T?S0aui%e9sSE$V@pRWF&#B|GvQ6Gwy!6VGkfAno$H|g?Qyg)WRk>vbT15HVH3Jj z>-cQn+j2DCF8p&=(ro6i2-uC9@*tmD-gX2N3RSNbPJUPwm3=^1<<%q;zTi@pCGk_2 zy-rx(+%n1)aJ96rB$pQuFnifD0-co3-J(z)#fzNEc$Q@96jjU0c!SlMU9wWgs3f#8 zyQDChM83jM7f{Pbccbtpcp~E_0)}#_??P5VJpH7LJcB%C=|fzlnlx2=_zqKX)Ha6V z{cdE~lrW)Oyb8}>#Qu;Oson`T5d_|YSKo~@2as$-ijPZl<%Mu;vjXy{vDfPkFDP~xrUuw{Sr>K7R{krkkB|LC*KGJmg2s``gn@6KK=}P0AFf-{61C4msuZAfk9G_=`^?n1Ez=V&WJed zkdsYXUQvo-edftQ==7OwdW~$sl_;CpQZu2orl8d~nda)F!jzNJ&k}E1i8rMr%KL=x z8Sinhci$#QJ6NAv@!Do%RBPDPU#Pb19A!A>;k?7>Ie5Wt|Iy9CO)~0B+9-H{<8()g z{@59u#nC{*n~pl!(*!q(S)S;L1B=F@H)q{PL;fg(WOkKDrg&wT?j)+ltQ`TIqZZ;{ z6AST}hu6-Kv1@fixqCP=j%wxLb0cS5HEP$0&G>O*Gk!LoD(LIEeD{p8TK=xaE%a}+ zmM`ENU*PO;k|RF{P)sZzS>m+hv@G9uWQ09ZxH2h!pamvnG=nM*Zs3WK8kO)INktJI zcL`I?<#r!&9liyITJMoXgCH`>*t%28I}TIFEC#O}L1|SPV;{8{ul8J7XPC*1GKUZ# zg)Lp|JmZ_Czf5clFW>~r85lNeSDDog+efS(X9`1e*o!iB=n-5IEVA_X*1O@ax#1W3 z5;%MN5?+)6gTpDW@xH`OugRd8KF-m{%|(aJ(#{+d6{ZEQc8IcRV3iB*abR|u*}B;B z{+r^n>I1#Ye>_d;o#0A1;!PE;v6&Qv>*zp|I8tsngyPmOemtP~)HoM-VUt+o=U~PK zOOIRCPg7peak1a`N7|JCq!1ThwldB44ngBt$Ay$uH~m6%v4!b57h7}%KG&-Elxqf5 zJ!1+(l?`WAsR!ElQO0~c8BjeIoh^`FPVqdM3b|AxA$woqT^-8vn^%T z;c-Bwp&2NM z^yG5uU0@{YBo|jy(74aF4IRS=2C|0j!IhCZ{I&~=?4+poV$n_OC@MT~0Y#TUSe!L; z`-W-aabX8wO|#}xh}IgpyojU1f5#d^PdHD<2Xf&#q77r6!hMO^mcJxpEouiD!pPXT zW|%cnCusVnuwT7))3bfZYl`K-No+fp%xa7}SP2V&SWu;k9tjfrt_g)2IgB|4184d{OypEP6+ zC_tdTfL9`4mjEPCdz^kI0}C;Wn+e7%dv8-|Q&i3Fd>SR#=HQ#cw*A(L-_xB^(0o7o zdG%t`B06eUh+)JBdEX8d!an}F#ytypYf5T#TvlTyfpgG5qkau>a|=FR^=9a1LjHB$ zINx^}lZ#MLeZ3|>*Nf+Y_^W-N!01Wg72910a^Bi|>MH6BF6z3jqI^L)8jpfT%H&~} zIII6V_4#YNY(|5iyIEw_tkX0v7ml z1*cYR3VW@KIZ&UgV|^c>4w@@Xhw=>p1UZJ& zYhMC((a(BuXei88Y(^AJ_p=T zp=iTeK2I$${CG-{-AgGZIuS-vj!cASK&H9^oA(`75#V;%gj9Op5Sq8Q!c$#tJ+=$0 zAD5P+U% zZfoHV{bn>+6bwWm$VwgTGhS(+*`+SRDy>xvslPb^Y!(LZVXlKs?r2lA;<8gZFIY|1 z(-qbW3kDkTD~YtGklEhuW+!F!-Ot!8g6-Fvv)BFx>#YqxY1``Akpvc!X#qy5E-1UlP=5>So6&2QXGqp;aD9wvrcDi*HMMuYaA|L5)xI|Rf zyzinEc$zIhieMwvT1&Vv!o-;vtT?F56`(K;fBPHkA7}t0zrRUZdD_~;6Iy&mUC)0! zMcA>aHhzP;3uK5UuvN-%{0vE{E1C6r*i|@X2N(W0?Ssu@iXcO!^Jc^l_xO>WLRX zpS12EB=ctcRNPBAt&5pGXe@KZ$HIita>IXvG}a;#`B77Ecz`$X)tdif+1&`92}NI# zd(Tx|N7HZXpGLP25*0f)t!&I<9Y zCFo)~UQ*Zsa>xQOo@o538ujV&-8u!7puP4kcT?i@YTpD;b)~hG3~oG>c&}LTSp8Kt zReq)Q9Wa@ugvU+4)O*3;53}?KY5ggMjlZN!nS(#)Y46%5&6DB`|9|tjysXkP&g78G zU#kf8)x+!0ls2<3MLwgVHeShzP-=a-c=(C zNVnFxDye|9YvQiO`da(a#jP$a2a;0Gy+=i*zt?TtpAT9Z^?iUIE1|~y{eao1-@z|a z+oRwfeuPK!#Dz^dV-9cUg(|GG&xuBTP2gs~55vGaA3ko4@y5Nre zKG@|ARZx+nD{-+QY0Q^&eaXiCxhh=-4jN@(pKcUur=jmsn@XD@X;W3&MB{$P=hAl2 zme3SQTbkyX_#HE@CcwQ;hCePo;tWlUmZ96Vo7z>onhAJPe&X)w!%a)Pxpz<-uM7P_ zz+jac^{-R1ClQ|`M*SlM@wHaQ-{cabVKey$-*J251r1*i@F=^H4Zoq0A&+zxLkF|P zlw{*Iyos6RIXs!6V}m6T96$|xo*ex=Q8Mi^XOEFQ8NAH6h4vWtiNxPah@un{gkJaZ~eh8BT}2HFOC%{k^vs z_cxGClNi7C9n9!U$ySn4k9RO}(05Yc=Uq|a%~R-7My#q(_40DwqP-OBQ`A$ax5^%S zXt$6=`n4_v^Y_E}N@$#MzwBBE_iSZ~RSJM0^@=3Rtt{i?EmV5noY`0ll>W7KzMzq| z?fD92_`gM~TM3v+o=i7cux#c}ijRz_s0?KZRBAI=6qlq`LZ4k~={TsWQN}@=$d`X5 z`)bK=Ka<}cAHf7%=$LN0!u53v!1ImQ(5(R+KTfVtZ4?kC;P|@9{=USG*@M$OEot1* zG==O>)(!LbCL8yq&@^iX)FiqbjhiawR+O;-Wty;yd+^fUj5nZ}mZ$Q7GeLnBv4^2)5`SY=ZH^Nx}=MY*=|_gthmslvH%Q_A0f+Xnje&ws6!V}+lWBP zn8=gY08Q~uIVSRZ!9wd0t%Ra;fg1YXB-dMxx4x;j9C#0wICz!^@Z z2uhq&;&qzORSR%On9o(oGtqob&XoS=**!A#!tvW~=31}k&;^HT?6cF5t2*BE zdW|{TRZ~d#1fdk3AMteZJjv6+vzrd)?YoXpuN=jTK82rgU1iyIk>4+q8s0$|$ok|K z(5(Y+n&IKr4|TW*77-Q+jQSL+4yKF>&O(A~O-V3q5m=8bs||v^Hp6VSCLm{Ip2WQ! zTez&9iF@|Y{L$B0M-X+F{g}lmz?-r&@yHP>Yj*}<98pbzSQj9NQNMvkTvt&^qZ$rj z4hn->k*B0#xW2ufmV^_m&YPuiwB~BFpA+N+bBUM_@E6$Dq??G(s{?l+PDUd6>V|zA z--9<*f%$W`BOqq*h8@Bz6QM@y7&bBFEC=E89-{%ZIQBEPzKs2=w)n!D{A6zrZ-FHi z%@AAC_n$*EjQV#4Yx}-t)ZeQDv%Kf{`%1j?{e7k0i~N1xH5z_TvK0=_F_qS%q>ny$AACzdBk+DL%atTWk4>9PD^@W)W9b*wvG!d={2?< zlF5?dz13XOTjM`9QNkQ3du6ndMngob=rpc4N&=(2Pkc@}@xeUdAatzEx_}D?-P=qGl1v;_X<4)HE+7Cq&-|D=a(t)5cx^ttM$09c z?d;rfpA#5Kx>5OyPOnY14%W#Udl3x}yR1FxWi&6=Z*x&_f{P`5%;Umt5i2+`PM>B; zj`Ca1O;d$r`0z+@^Ln=aIH0tSVJ6Ik6NX8?^^~`CrwFCNQ*HBam}B!9o-C`P82Qw> zbA%~y*ql3OX-r#Y&mQ=`c_Q|mmfT#MRxGE0o&!C9^&ha|7F;yb<|G=@r6+KoQ;J}eVf$Gq)H@x?tI#wLwSn;7cR*X@{ij09{MOyS&;ffwB=KGR* z{@QcE|5w}MrPp)9klW5Zd#K7u!LH<8{HFMZ@$Tq3%IET)qv}-S$`Dap| z#yN~mDMg=^vTMUYnfnG(O7%J=Vp?^-<(j4`1H_u~7zXQXzDb@$YhNLUU*QwPB9pE8 zY!D*}k|bsf2H{({VB~Bg$MD(Y5YM8^q`YZT9-c+{a8Il#*=Af!mM?TJdaV@0Fcc={WX4aqC!zk9k^H9V5B^>+Z_y9R@!o?^yuT3jR-Aw;|DFvwy@x*C^ zq#OGL%{A)xP^+;uq2k?QTN%%V(dTv1XPG=}^2e6&^o=FKH3Z5f<+afiSIcujWtf>b zB(A)?3|6pJ!+Oh?cvAUyOp=VYXvTi(m-o~1q>7U~edCDeC7`PKG@7DYo{=?amEqBR z9vn+0#Fv*_qC`UZ;cHQ5z)yJIAy4vE*Zcc!s!idh&olh`ynUX8*^A{Sj{lWUx9iFl zAaELIy+lrBNr|&+S$QSJiBr$tsq(7K^ap^LIpbmZy>?8&Uuw@UczONB1+Q(qsNm%d z7Zto#dm%I_daYIP+WL z1|}Cwc~FMpXU*Z(f+>$l@S*1L)^hpJS~Q8Bs3&*D!-6d-qoLGd#1u=EfhE|Z>!R_31KtBQG2#T+P8=~av@Y}VXs zq1r{e0tzSm3QHn>g*1^BRjlRK4eW~CVsL8*yU2A@$9cVvJj|fTnHFwMV=Z4P&6^#I zh5FsPmdHx$f3NG06qZ>&nF8)dbtWI)H1~spw$NC9D=V%4Ad$NeOZoEMikl&nT$rl$ z!UNSnDa?sPVx^~+SuQC(Fjms9c#@w;dKqHFEQGNAjQTdR1*Qms-;^p}r*n%~Wt)iA}gMz zEDngf^gRdpT>jb)*g_jxnjslSs*IdeI?Y0dG>N+7l%K<(jP1N9pg)JqB=scWs`;u2CH&wOA-Or~&HuLD-rCX|zo}*7 zL-^&|X>Z4gS80auZf@P^+RzKWG+mWiQG_5F(CK-wYvzj|$lejFt5yO7$KU>LnYpfTEJt4|^4@@y+=G zw-gj$rb+@(L_KdGv8lyYeQ`o0RY+=h*3w_h7B zXLlkT*chL6=!R&Q@L7MPwk~IwtRobY)tSTY>H>CKJX3hO=HDLM7+oLbyt2GU(idu* zrG&)atjNTJ+^DvdtbvqqpCJbQ)J{e?i*j)iNw4~nEa{)mg4hOm-v|MKy&MEI7jp5Nntjl?~k+#tWLfI#2G=oP})2 zKJo9yeqc^ZNkMyXW!!o$w#?Eju1t3OQmNduLCG+sUz8;#9Bw+_ShEqtIt=sW z!kq|BvQs~&%$l&8o*xiL-otj<)v>(|v69|!0zY{iJiPyjoRlyzG0K=VAVyIyXGPQ3 zs~RaBKe?pJUP>JIeHn7Zd5r`;MJ{0+cqf^qnR00TBXI?N#{Ez6#9H&1idfDQOJpCC z5&BO)G(MB8|@v@J}FUs^e%F4otidQ;@gsQvhHomvQDnk~>U2e^jhq*I& z!|?$&L+x$UsIQC_I&mQEvL5D&HR8)g0!K<2V*gSgRc5^mZAJ&}i%`q|Ir#>7O69^z zDoRmYq@w*G75C?Gl?25u!npjm*++}I<6-crHWT=rSGmCYrjG0lkK~@G_HUqyix@x7 zGtqv@lD*LZ4sUAx6ublLfDxB5M#GO~lzLE=i?0gF>f(>YZEy;pOq7|>DVQ~uuN$r# zG!5p)rol9}1)rM+*RMZ&8jM-_Mbn^D4&yD1rJ4gUqOv@3*pZIP64eXSS&)b0|6$Lt z=m%*N3P?u1!0}6`=CRMU>6JU9$b8`$WNsAHQph}P0GZhp$kc?P#c*A@KsXPFhfGc3 z3>-r)sWXP^`>m_1)a+tcR$<*EZXU|5Iv!@x0eA3*T?3f>)Xf8!EbHVhxS-6Aj(dHeDgxiHi{h#s$;P~-~7o2@j6&VL$+J(bw_S6XLQ25C6d?(4WKj$N3 zD;yWc=-_oC^dt6=Pp};A_H~Fr)B#2dTd&(WRLdcf;WvDz4rhm^FzU;R zUf~taYa~6B8s+>XqKaja*H_d@^rKRV{w8soRot>6agV9E<%8oMy_q*)p0=X-Oxfgr zbOrJ76(i}(Rb^_rk}e5aT1~U_eaCSb;iw8dPp9O?6;Zv^j?f^y?>?eR7+Xj7f}P&^ z`7@*@Z!u*@rbrOL0H@O9u#Oq}ODQ`ivX|fUgQbTTc$CKqj_Q#`7IBH9kj)}c>q5}e zmT3#;J|aD%+$_o?!PrQLu;9Guk4W!`M_xqY6Ds5b`J%0pxD;p+T-0NHl5N_6P}SlU znyzS0m;SA+FugG0Waqd(D+TAod+qH7{cD^By~bTD*!~*ZCt9N6w>s(L$3o{4v8k^P zfEahZ%FviQ^V)xfPMHmb%Iy=&+DT{J^;hAT=v>$$`z6VGc9IYD*d}2tg-iH92pik~ z7QZ@`JDrdtxW*Ak3T|`|WnVoKYgZ%uA!l%n6H8h~aHF%Qox_dI`+Z~kdv?SB2f|q8 zxb|dl4T^5JEp_=rNbcV@H{oH?BW#*eAA5M z@_JilaAR)m*5H~Hr5?(+YkPxhda*r0@T<$Fd4tN9w?Spg>pXMHik7R9;uba)CTA3~6Se1gBvVB)1t6|T3WXlTflJ7jPk>_FJ zoCGHEH(@Up>yw)1#x)klHO-Bu+{WT~f4iMyyp+CYpmbn8kXEINgVw_WmsGzPNVQp~ z@wdlG3wOrJ*T#Xbl<_<2e2BjX_`8?Cdj8h)=jHEa{>(Y|9Sh94{}@k;r^WL)&*MD1 zcy{qT$@3)7UY@-?PxCw-bOgaTo4w248wH0MkxqB;cbLB}Rd!a&((BtcOOH9<4TN>? zqcOB`2t)u!EN#$x+3Dc={L|4v$T~9!sh@q!m2%dWg&}$n={X3b9Rw*WHYR;~OgMsO zHB?z@OeFE<6@>5Q?`i&;=GrIhZ7jC?PblbpprH4MK}Ad3qG`^I)rolj2Wk`=2c_IR zC>=eZhtV-RVfTL^BWv7+a9?Cl4$5}YAQyib{AKc&&0j8mQ~CQgf53VSupYxRgC{Ki z`_g#A+`rGolWT(eQh28Dbn2-4DA=429o z^JQ{Y61DMqqHg2z7;06<&zfcYY*@*M8;RR^8*v-g61Q>V%HXqg)DnDAey{dD$}%$F z+}rZLsHvYDl)EV#S*dSh5_6wRaA};`MvC*B&c!KYqVdAM;;`M5`<+Ht;k<_38$;CA&iQorx>0pAr>sv6+uLoq zC3REty*>T5x^G?PONh*Ngh#~j?BVt`J?b+quY6m5)_H7EhtJMZE|l-;7(_h38?nHLxe~~aKyz{_b`?{ zD!HNUD6{0I-|ZHw6fj#x!mADXI?U6LKVI{=HR)CW?K>Q}%4xh%^mf3GL4eM{RSvn% z;2c(AoyZb^WznzE()_mT1FKU|*ja4tTB?UT<_qIIIKhLrXWXAI%8TUv9ik;SsOiN; z|N0ebHoLpCuTZ-gbX}q^UF-S^SEtwYWv)&%e$(;#`)$I`tUEgHbqj7{-oB_T5F;kW z;CmtyT7=5{s`GT$X|J$JQibhh<*bz%$LZ|N@5ZQ3%SfR*;qzf`L6E$|tR&K4IVHjB zUn7b0nxO-pcxyW+UamH8UePR_2eQnfApDZ(&-kcbcn9mLc>T01EcYM7(WGLC%wm6$^Rx{@hqGW zD0)Dg3y*6#niwd3aIlLq+>K2@kn?F$mRsM?pxpuAX_MqB%K3{~^kDa#I7gf>%X*i2 zs=dDvB@`zj!7fEmCtb!>&uH^R3E@;&zv!GA-YVWms&w-RZqt9tb z_)?-f{Cgo@Nzd6bnVAXC*;pHD0`n6C;{w++FDtQ##H%BK%+1>Dz}iWHwV8pnS%I|~ zfwf~KqcgBJE%ZZX4diLFXv9g|9lGtM4@RzlL%f2MmblBAP_#KvoEDlxVpKhf9f9KE zQbDljVK^wC&$L{zx{UfVQX5-~IXgWZ=S@n@;z$urNPU(^O`sV4$EP`$_-nV>q#@F> z@OOemjm+hzIpQsgmazv?ru}j?5_>#)?tGS54`nmg`FT@*4$IG9<>w9g*znu$}E5FuWdMrseO_ zolnH-%vN5ts-ab64>RkAqWG2aw1SU#=C@3Qz3I-7kDq%nq!URMEgE0va10p1b5duwS^HB)| zCn&>q)b`j`Pi?DJ`_NN+%Bc!gttLQ|fRrz(qEefx)g6b{s2GAG^ZxF&_fCS^^Yr=S zx!!ZV*UQDuUUTpD{c*2*-Ru51ZhUwbh*3MP-?J4Ucykaem1zDeRpIXQR=cXg&0dVW z#4dcnLEVB0Rk#pOf6IC&*Utd*9}|-+t&?jQ)=^bi-gprpbbQYU|7Mf zKNo#7-yu8zwjk&@7}+JQ6K0WThs4y5S)#9MH)9#$6)T94Tr-jFB5m zZnu$p|FY0?Um~+^-I;AooX-j8II>NC-bYgeHxpXc7!RcII?nKy8n zU>bYwYm%@lDi?A3l&D;zWw&yny@U?JeVyFb`C96))pwnBC5PXpM^4rsk=@z8YUDB< zd8}F~<5Z$&(^3hXxxn+lZ3LVrxp>u%^vxo;WTAOq87GqgeKXPek->UbAqkAaKg&F2 zw|(UpeJpggh+0Se`vS%rHpHtSZ2{wB8!|v3>RMA>jLO|6no#O)GhY48tIcrJ$Ig~V zpc6i&RbitL86c)Y0=YS%4vqvJ13!Jl*&WOp_$j#kqqmvZoor$L%txG^Y&cT}?wwm| zyw(HG>I;5Jz(vDI;BV8v;7g?9xZXTaLgdWNt?q9c_{B_kLW;)ZHX#dPVr=mpixeFV zolh9vt=y=G28*je;~uk#NUHkxk;lL<5czro!LHBneQfSRtf#btJv^S(rDqcqZKF8)iO?Zp+IlN~CQWhYygJ$7 zA?y^Pzwx*zB=jyY7HjmmPH+{X0WFcA948d078TC;pp`UV@7@>}b)NP9=!^s;Y9gBq zVi@DSG6#?m5gE8O<#h3vrVw20g4Ruv2T8au3te6se@+tEnE@JB6<(h>Sp#1shr1Wm zy2IVyU5%ADZ**Q0UN6Uymzh)e_mU4(c&YK(%{0rT6V;jE34Zt7Y-Je_48cuvQ>d)^ zC=AW&-9Ig)?wQmbY%hFMy$$@LJiZkpJsz$+;Sgajb*S;8DW|4gw;W}lOk4d1WJND! zA3qd>MLjId@5Nkb-x~yVog0!40uKf=tJ{TKn(!&}>EsW{l;k7vYETm1)Rgf~}! zoABT_yin~jo=r|pFv!I#J8zrXu$R{C$irM(Qm(%}WYr*?IGkCOB+VqURM<{?8ziX6?ph?I24Jg<;h zUMxvo4RV(9XO#vs79cQ&40(1-=xUy^JYJrGVIxn|HnP7u8k85(6S4DK{U?O`8aoF~ z$rwURC!|p^XQbkU-u$v$5RaxXu>XcJp^KPd7ff{!A#|>3lX2Em2Z=Gu_-U+=<6~mb zxga)msw0UQIa3`742H%`mE@t(#t4v5PXn*#huOe!$%yXmApxd*cOmQaLIcH%WWK?? zq@_~W;KwJuIm2=C;7I?l#wLB&nq(+dB6Kz9g$4<0JOMAG4u>mTbizd<`hawhrJ;V< zZ6-ZY0eAr@%ciI_cXl;OvM7~4`CtKVRQ&~$m;kG z!v_Ws3Lh*v?SGS4?<_SQkmOtwnC^H%ZOnp>y9Zm-|Tk%lPZjA*s_x~Giv5$|5xT6cjIACgNv*!dc+D36Pm)@zKFoYwioD219~o&Xt2IU{#U{=j}s+o+(_ zY8bU*KFOLM!cFdI6>@LfsDP&%RKjvU@@8PDJG$6Pv|_eEz|nsq|NO(z!pnNm^5ef&%~sTHno*o zI*N|<4?6Lwd0Z+FC8jrLw{TKC?rf1}pIp-0?EA<?IbmJ|Y;(pV`p`hchNOIEG$o*EJ;wJtE zl-6Qj;=y=Tm7S#sG7zY;mZHCb$@Z%7252K^w2Tv1YWUQxp``Gs%e9oM@ON1-1GZ*p zsV_SkIAsb&^KxDu+`RUblLtow0!cA#g5%55I*rn0tPDkUhGta1FSoqL_vQ%^$R3tc z-KhW#Hef{cX$4qe1J0>-&yyqzEP#e`e;z)d4QIH0Qd*~kk}<8jFs;R89gB9xsWaG! zSvg0Lr7(D6li)><^7R+aPTerQS-3gfNicN!a&yA%SXnZBJ&BA9;zC|q>}W9~gJB2^ zT)eG}y9oc2>N~?;i1pY8`~%(C~o7xEXB4menC#Atq_y(uJFy zW_8{)cfr`9C_dcr(p*}G37%+C=G;C;P#{J_8M#`z_U~qzr*QWExGfCuU9GWib@Ht0 z43UN@^BP~^$fIKg?UiW-nQE#uDjswTSRuSSv!YkkMDa~U9bF_Ph3Tmj&U^Gm+@cqv zn4TFH(=$UcJw-h|1FkApOQYQ1qMn{f?CGej*y`!&=#dE1(`_<66{}M*J>4)p(=C;f zUXJGiQ+Zv9_I887-DPr!F>+8jwG=`I(MS$)u~4y|RZ4%{8k|~eTv)=K?u=+BL@EpVLmpDvH6NJw1NSlgqdd2J{nBjx3%(GyB3S2lz~UTp=GNL;2Xm zN3%-IiD!_e+!ZJ`UXY5YgVLI^Fmlm1#V77CM$V*xhzONygCoOsaN-MJ;^?@S38_!t z;flC+6o#klaA=>ZqvdsVH9=Qt{3@w}@gU_%bY&l+TvaP}ljP*tbdcnmy4EJ1d?#9% z=xg!ZhXzc1D$jMEFq>375Oi6Y+(0HO2Tvm>PL5m_X#&QSbBU@N65@u%c+UP%YK-V_<)ssdG-zkh62|QCPZhL>}`BMFpGL>Qi^l=$`h}-B8)}5lK zkl5JbN|`b>fVt4le~0;7YaQk0Mw{QHNm zw^xYqU8t6qnKT<>)#`;2oO6vez29%)d#p&W@6>WMcj% zc3FJ2JAk@Qbpi(!F6N3EgQL=!Uh2@_7~7Mbm>N5C7V4F5v{m%f^&~RNeS0v29_(Oh z_VnNsdQi3!v5M5M*yze(8`Qf&^*_6nRv%?fyd4_kJ5zT~_Q|MgcqylRWAAje3Fy1j z-6&dPV}8TZyZK!;6ZKX$GX12`FCQd3Cn~Zf=k44DZtaR-s&Ow1O4N7c_n_kH#}EaI z79v#p3I{#rO1#iE=WSdnt6lO6e>lAPLwjEgH>EpUIkYTeY`F+FMp3A6gvk1bX=J@C zm{skN+O+(YI?Bz)`{BxCj$oSc11VXCFs-FS+_wPw0#$pZ*QP{j!(ZlkqDuRA%1?w? z5lQHmsYIl{U3V!+k_f6H<`jMuaMuj(QRas?z<8QS>HjHtafw3C4h0+>bePSXLG#*^ z%;wE-3unc+T-xS8Ms3cI^4V;TWEi^0{JoOqgKHdIz=_-Xegs{Wh!)-oQle&2zq0Z)bk~w$u^xw-cbWI;X zzf28J84wR$6P~*57$RdxnC{7L4rPcgwl;8X@I=f#H~4wXIXBo9AGA(*PA-);oSqCi zj14wU`enuo;blBeP&TQyc#q`(MV$>YerkR(_Vz@MAv(L}Vv9<<@S9Xxm$aPAe3u(K zZK0K1?@4Yf|D3W!&jDNp$|x=)yIZa7ZsvOPRP&cT)7u=9DJgMnveimizS@Q14o4{C z`jGp2ZD6T1v{V{e+S^c-E;13}&;>MfKyTp(WR;o?op0Bm4|^cooNkSuvzZwvGfu_o zdutElMKF?rbLT>ZgeNPy3t#aC)j`wT#+dMa;}RaVGwPV9bsm5zn(dZMI(0sW%q{Ime#vnx1i<>z2?&iXH6P zo&u3n(b+XU>pYjXd`pUF`_b6-TbzwenoHjs%V_>9N+3;G^Dn@@DQP|1htH6#b_JP4 zDw|-XWo}8C%}(@r^Lu8Bqcc#{uJXO9ohx@T1N92T4LqCV9x|3Z^l>LR0vdtFf>KP)Ka9-()Nj_1y$tLQ-0Rm*Ra zewMDPmajO75z|YQrmStAZ%iua>8+b7J4`@6a2^=$XkTki8*p{4Zj zg`VxsNQnvXY&wUiTbxz;r}R%pyNrgvdy9j9OCnC}rsfaKZs;T@++0+NHHgzyzVD>h zrt^2e>f|ctcQRs^6F#?gEPpdsj|4I}b^%T-PLn{2%?Nos+Z)~IQIaH<58Isu1S9?o zN;PYCJvF1qjM(j4oO2j}6U~*URnh(Yrwbs6xO*ddl`mhFoUcmG&xlO|a&GOF{LNfF zuF9D|ICe27lnl>~gUpeVjIQaa=Vja?;^vfOR{`H_l2eNesfcsaq3B2a5&Fm;(gh7D zpqXRZsf1W;A#{`K@HBoT9aWg_OUnEM1ThMp(Wycf6=wL7vMk75CM46BG|Yl5H6dBP zB(DW2GaaW-E2Xg6NpXsh!_j>BGTJM-7u0OMV-0&)AWQk zBv;=iZ;yToi>oiYN!%fZhJv`UF|HF%G+PfglUZX*krg}CfjYRr_v zV(&mghF_F?;aROsR51&P$;6jE#DmR1;*}ZatJnv?V~`2DN)QMR5+fv{v;3pk}pt-@!G66RL<`|B#-=@q!aIhVFX)ib{5FBEIZvq_N9k3F>j6D$uX4xOF>P_Pf z1Oqns!d~#GKya=NPU!`Y2?Wo_4zh5;j85*GdP^7^2%e`BKuNvM7pve)1HqeZ@GHIG z>_D)@25;>Jj|&8qDHb8ezIRskmj!}lc7oNt3C0J4H`(CZdcl_mf*b?rP4cXeLKjB{W>?&_0X35Amqxaxk;}v#0cMBQuiIlIG4Axi18O zOpvHq6}s%OJ5uPj!xJKf6YTKBNZ~~4rVQgHMm+kzHIN&Mg9u+H>&=~>=0v?<4Tyah zK{oV3@n@ts<0={uuUlhW#v3w?3<}mD)AFLEj;JWTLT4s&3UNB~CCL&Ly!81bg$G=| zq;w1No(XaLk}@pFArmsemo&+OykbHo`jVztkY5YrY&?MNJV65f&fKtC{6liX3K@7< z(tZmGOb6$SL6$&iMC&%La>^4 z2iRI-INZuWDTS*{g+0o9zf1^_i8ZUI0q&iU69eR9eHo*^$b=j%wXR{*0Y?4oQrp=*8>)ySc&yNhfb> z&Lj}*Ggt0fch*EcGyPr)d|akXB>8cfC(P%^Wp*%^AD217Jbqke0&}=yF{Dq= z`P z_c&v!dQUK}QtyezW$Hc57_Huuj0@CziZO_Hcz0fWRi2TmK+}yx^`2>bah25QM&ncU z2Fa-R&BnXxJB%RtM_fjUiDsL>{Rb+;|29zX8cyYR~rAJ-XY`X>b=JJclBOp ze4n>eK}lB|jRM`S*6vyrlxqy5Mg{Ma;9?bgP=fPRP-KaWA{BgCg40w`&XLA7D)^WL z$El$31&oVSaH|9dtKjnzOjE&^Bn- zp9(UKCB5AKN94Eel}R^Mf}!*z`2fz5?I^NCKJ*HiiH{8tZ>IwKPo)BoZ_4KKK5p!b zja^*HDJhYArP|RMu0Ze#Ec9>^=@t^~tLX@3&4DtT8@y7$0ux!yv`%4g1aoZIpMcSQ z0?r5oC)(hbt*DT1SJ|-tuwc+`SKF{30xJt%Q%riZj7=&(h`Hzt=(S7jcr{j9h_!4R zc8djrQX6N(3N0Aq)nzv9%HAy5CdS*~p;lBjh|6tQiUk`M2%?f;*66bWv(=DP({0#a zfT=AeIsDahCOD%Afsm5u={$mgoe7_25VN-Kp;5FhApyS zHwS_@*svlCHZKso(T3&rW_fEMSY(4oSW#~a1ZUf@0TygYAUMZ{eSyrQj6-!GILU^6 z0L*ImvOqA`1|P7Zt_)z5-0HU%Em$ZJ^x3fgWx>`2f>UhRzx8IhF2FsAm3*BU_1*?% zZXkHA4X(7ng1ZC3>um6B3oLj%5X_THDWNQ{>B;+ELF<8FzMbF#E5U;bJk16tTi}Nj zxWEQ~oUgj#UP1kV;20Zx6tLMFYEcG)W9=_5TL~Uh35MHXu|!Xo-YYvqAn3Kh4_aW^ zEdtff4A!UB3#)odcv^kzXMcp=7fbY#o9rys>J_SfngxDIfz8zmINJg@DX_VK0cZA> zARCaqhWYXZroJS5*^KOE4EQ|@EW47uk^%3tz=CN4tY~Qf;O78aU35f!F_$!7erzR> z&C6cZfEz9FF$FdkHsIwJxLtwGwGFtmH+$LR?B&guDOLj6@9Y&0c(esRsKDkT2Ry(6 z%RXqYbHFFB>#2RG`eH71zWkXl(m5v-*j(*^TP(0_m-d1O{2L2=T7k_q5BTBU>>VIs zd)f14y_GoyNw$7vdBsxd%cYs0MD|(vhUld0dS5578*cMgEcXS^=6-^zL=f9A`moD1K|5C zu#gruY5=TRU?DU5pa#Cow@ScZf_-1q#}1BIE-4k40AI6jGptMu+<1lai;@dcB44*} zbCeq`VGxV)^z}Dv{1Rgo8&+coW@wKgmJ6>}PsYbPw*Vc+hUskNYt-W`o-FGVVvSkgu& zM5JgRcc9gR$Ku;%BP%v4(e+R&aGu6L0!DLE$)u22rI?FNC2e0d@^U>Xa(YvU?Ouf9 z6V_iL7H!7q^+%&cA8$#3>3o4_``kDz>6p94HQ77OvpvCgaQz94uukG2|7Us|u{|k6 zsuD0Oq#c#3{=;9X86e#wdRq!3$Fe-Yl6clv@#OhqPk_(=OO_Jz_M6+gpryQ$tyQ&d8bC7^0 zV}$UT>ln78W5EsRLrPue8jmZ;`+Xn}2;~0FB$J-p6Ds673Ti(=o2uUak|%PHfXy~U?s-s!CCiA)T~vQm zuBbLd?vea-bW;jLv5OI_G_I9wwRYob^*&~dSMT?Yi+Sg~O(_L&?WmE(uX3afV$;ob z$i2JOh{#W+a5!Z@m8P_+;@6bVASCpcV}XRE(1wMC3Pbb}Gw`$$kiUC04K`Olm-xa| z6N}-P7tcPR4a)vhOotD^WU+#!56(Ui`uoBKoA#_t;;9PH(m^)p8EYIjy-W z_neRjKlG^c4E5~Dz{R&H^^~}Ak=~=#^Ks5$F#?mDp}oE^yLrK(?)fS!%+v2gvw77y zu(Tz!etC>LG7q(%nk6X-ghX+`E&j(#XNCrOc3dg98~v+%S8z7lnMi!>IP9R>{m56P z+IV~$&F)1!($wfC(Sr+?1UAm*{*qBWa*muG(LG*&srfBn6k#AOFJS$O-VBbEpKCu1CLbuFwMBE8bA55SWx>Gb)xXSG?8!L?? zhwt?*fD(K29?_Hcv)N|eA0vT4-sI%!n-fF8b=@8flcwS>F@c^%ku1M9nZLKJ>KAVM znbIqBPK&*(@3cBvCSz|$%jwx2d2S8OH-7&d*f=lxu$-{<8#1Gt)&V{FW+{VpnS=XKmcd&|t|fq0$EIF97ND~Y6#lfGxuR`MfXm0dtDz)&)J zk*!H0H4-Q_{@u>Z(OA;Vo74ch}<1wajSsIcP- zYSC5TPSkwiyO4XjFkYA5o7u0>Y@h?Fa&q_vcVg@rVkT(kV0Von5`Kh$6Jn3*1@7O8 zI>5+g#Eu*9Ry~nnW^?k*>IQxf-81q)U}Lk?o~}5C$T&Dax+1*TEy5nL`MssSH{NQ( z6q+EDAt#=DS4QYY`9*E3a$l6p}Zc{U3HvN&_J2PaQt?no@{)mXZ*&WFj6}t0c z`DWAr)BTOhtiBjgX8iCuX>D2Wg8eA~v$Yq6S?zjZPm5{rLRrA8y>^4`CERkPcL{@D z%9Js!xz(99hSu{!+Nx}t$Vjc3UCcGj%&6;{N3wUK*N%0rFYkOc zJyMj89yE8!@d-xFIF|208jngN-^DLTlgru0M?_Rp*Cs{l;s|8jmEb$G{@MBHfgfIo z)TN1;V5XHB+y9G_<#N>=Z*!0$v+)7phV0-zB^s7>5V=3(Pbuzzg*h?sFwwim_M{ zq#Fm#bjMCHmaLrhsM3l?TQn?-hi;2AwN3t_r|~#d+IYr5;sYm1DlF0;o<=#3bNP?# zh*!xjS}takxiUOeieNJ04!!i+eLPk$?O6kxEV*f_%vH9t+ChQYit zCG=Ko97B>^P54j^DDGzs}q>eKl2!ojjzU9Z9;Hy z5W+ZuPWy#DuE=Wi(_MOsjl5(jE3)QS!_2kpl9f_w_@!!{@$D+JyZ(!wwzl11LSFT% z%h>_%*+w~>oaCIu>g;m35N7w9l0eK@7a5q76+APU48FFNUjj%t)b?)9MjpPy+2)SjKvm!=`Fh$U}y7 zaCie6RL*Z-G^@_*vuM7)O`ZP@r4LMzwYnI%yVA8^9{?Ei7 zrmuCidt>vPD15{Qh{XhcCvR*%#cu@Z!p#SpKX5rQ8L7Os>@(%GdoG7VH!14iMvFRl z`csQK@LANsk1c?9QO|a-2!r@aYa!KFKlWvqAW`vd5iF@3A)N-qj} z;B1UETi93%8t2on8KXq~FN~|(G2;2BD5;Ns=RWj_E{2Ze93aa3 zB71P$_^)3=@uiyxgdIj1fLc35v9pXH@C~u^$Y$9^kfTc9Skx&8G-XjTdQSthEs_(x z%Y_Pc3JYc0oOLr3K4vBQTF7Y!Yn5Ig+KnMV6B;1TY zKu~-Ymuo8@PbaHlvQiYp+g-9XgBwCcD*3q>Z|TWENQc;PmBDPuD^o=|MW+ld!+NU+2_)QDP}F@vJ?3e-Yz?Gq}MPhKGEldzBV zsWSY0g=kP@h-CQ&`AmQWQM6e=5g($yD`1m|(4_&KSaw|_YZ8-TiiqghlZOmaYfxB# zr8#?Eevi6FT?2&3EWrS5cIF=@$*di?=`aN3N+BR;CiwQRKL(G(+zAyQ_C+(9Pm*;W zIuk63PyBqIhL!`OC(rjFyMbh}K(bh$;QMm@L9)OpgX{BKi3m-j_SMHv{0)tur1mDt z9GbId?p&B?XK^~oF@CUO*EgYt+QuRG9;J^|UMxf^M?cJ0?A*}JQ$;{NR3u9VQDM!s zscS>vwQa#$U^u^$lG-^HbqH0)!*-&_-F;Gsu(6rDxgGi^*^%Cqu%%+3q^$eQ7txqt(eco}pD^XCU| zxh4CQND4a%d3LxwRlZ%d@VR!bMyS(^Y zG+OA!!;r=9i2C8dZn;yk%FWOrpWS zd|;|unpID#u`|w$QW}Ep;9z4Iy7T2-NiR`}#Lz{-AqLWwf`^lWDdm40%X%8*aO4G@ z#+jXJrz13Bnz@ksdS5URiVicUj2k8#?YfxU>=?!Y1X{#=qUgHH?UL|!ZC3-IVRk@V z4g8ZErpqn_ugeWnTkj3iN9WKJ$~yZqU%z4cxnLn#h{k-%G!LNOvNBx7Yp${~@i8FF zE*KZhux`^8j$AOszAqhpmUn^So*ul9HW~dVs+kGYya=$WMVR@Gt0hK> zltMs=MLIl}()`dlBDb2UXrNG%@=Ay1hq}R&OF1dyoe31|?+iaLry#}?>IkNJvS&M% zz!UI~r7Ua8)(a$HY!>F?A~DKJIu8r&`@-@In0rdTQ3sxRwILmsL0vdFviaT_D<%8MuEN{*LGmDMh<7-BeoJa^p2F&-Uig4Wva1 zcgyr^MkO|RUWSjBe&`GANU%MZPj&D$bp2Fic;ZrCiH^`nUYw8u4SO0klZ9?-h;5mE z#i&sm(-SuHj!pcQZ+E9B#@=oC>jp^^8)!!`wdER?Oc=#Am|I8>Mo0>+*md%)p@%oV z>u7=ox?5_ac9XMiM(G2H3eS%GRO~Hbee+ zV9N?3-$CHRhN|!U)vtaPOq}2G z3w}S|2Es!BGFtzBq~zFBGr5Bc;kL5XxfllirF4BKp){)0|WjSQ%|LaPal3u}{n z{7B=}IaII6eSCUKv0R@AkBqOy9?LA_+R)bS#c zN2euVx$PV}Mc*Hta&LB9PSa<azYPxWmm*S=ClD_7ro{XzYJ^jFY} zkBfQPtwAS^U!dRolTJ*+?tJZ|w37CbG`^^fUIBP*b${VzFtCXfEMy5wQ`5tpcZjE+^KXP5a_CvjB9~XA2Qs!_TIcqqB!{=o;Q!81M2kC!v zFu(t8gPEcRGqy^v->0_*hd-jml7Za#p*fOs!Ux)C$Z%vI0M)+sq5f*5{+N>?Y4@RR z-)@cJu{yazGzV}D19*`+fY#`>+oRVLY3gXdj(YSA+>f&N+5^{rALC@VRR(}= zJ*%xapV4={_96XU(SGs`Jm{8ZzbfVe0pGl-Q-4)Do$eahO3z1brYDPz$%JfYzUC3n z*Rt*nT{}wWqYB!dD4n14cDZl=I=9*JeR}<)j-#CBeEplKk>|djOK+;$F+*~iScG~r z$6KHcJ(Gp}&#Ehpjuf>8y{~;J<0JK@HEM29NmWB63SMhxcAPagw0-|l_4KQbP_vx3 zomWW>3p#!vHB_*Tl`yU18=Rk*6*gy1L~Bzs`s*F4dQ$(4p?-5-_AdHSYDs54XLY8` z;8wF2ffdq~;WJKVFcJxLqSl6zb8kN$__lu3>zibTCdtrZTEhR*Yahsk1`HubLt5wK z2LY#HDzZCI!@1cBm|T&R7#6C$T4LlHh(Fsj^Zm&^8_oZV7`L6;{{I;Fzr(o4WSGIY z(o_FG#?}7EW8C){r~g?P_b4^8F|O3^e-6eyPHhC^N)2s{E3iHoS1SBJ1>>r^d=uju zRPkTIxWf{9Fz&|V4WR2jTn16{Zh$P0_#ei8Z_e9!n2vu)h}KBOgJscrgaV(9R6G`` zcrsG)Xr$ulT>mHxja;78b22{%nG#OI?i4nyk_>%sq-figRLr7>JPoJ%g$SjFk{V9+ z*AP+pa7*g6%|*XbMt|>uR>SE>c5wlAfDcS@uTI8)@T}OMaV&=u!-huL109pyT2e4+ zUaZyga&y?oRveE3T=ID~-A{61aQ-@&_&WzJnina0F!mUtJJdQxLm&|zQ~9pi`t=-L z@J60)IF%jBZa8&W=;DS`6GG25oGS3#KbbrmPWjiTH*_p%IF;bJcPxNla)(|gRA#by1 zMl7L_ejx_5zxK3_PXJ88oYe{FAF8agJ&jTas_kin4HNJ1^T#Y8;(wCtIYbU2BaYhs zr?;feluSb>BmSrPBsCaq6+cLdcU#5JwhI5iE}X9MG|Ii4D#xh;1?#bN@AusQ4=8P` zckU4u->i;)s*REtRba$D_a>4e=<2wf&(UdTBK~dW_@)H=bqob&4dn=!R~c$34+9Lk zI|j=u1-Ukc@}UlqaSWz(FmKHUJ{ohH^$c}4bX?JZrr&rI|CapFk$zqKC+DUdMMO-_&`t!riH!P0RTan<@V3sr;j{;W9KickWO1 zysZ5y+;Y(Q$;LB*OowAbe{TomjPlTl7)i2qT>+3D1>bpMkQ%GQRN74>{y$sm+Z%{)=r!d zm}g~#iv5o%=NKao;wy3_di=+tt|6WsoeCJy9)dOM313eMRbtL2kHGU^y&}IlJ@;NI zzh1dc{X)*5VqeEK5)q-7QH3Y{`vGQvJsV8YcSTE%E{yo^i_||HseckzlJlb4?&!oqq{EpCSUgq>>QTK2 zw71!_!{PZ22jC~2vrhYFo#yac^kf|_>hc&v|K`Z7(=d0ORq@2|@<(wJl{jZpe8cbc zqHTKp!@B=I#3A$Mm?Z)KzOFr_mmJ&JlGsFV*B^HXuHCM{v`iqR|iVD}R=%Z_ZA_Yb&r}k5i1J%6`Se&dx5J zeTNbXIGkYYv3)p$b!hpve*SH`e_QikhadX*&_`*f;>_8SW0XSn+WsJxpeSH|=-euX zGCrd!t(6`6x-2Q>1{}GXnDH-b+ONg*7;kJ3KjGc{hi+l zKP*!57WVtFjTk9;r!ba|`=E&botTR?_ZG*ISPFsHrMpU=R?~vPuN{ssZABZl_1SoK zBfNwcU2t=fqk*CH?~6?)@7L3`$H<67{n3cOB~rAvDx&R=dBr}s|FLf*AvbcO2wi(t zY^n@o>2s3w>?i`*qHBAbjf~ipI5>V)-^J$lT6AGTY^18r%TjVMF_vL}ils@Epo`zH z&DZN2()vEGaz@F2XXhBDR<4#%DA`-S5!ExpAb7m+Zob7Pko zcYO5~Mu3fncv%*0Rv{Jw)4bz(aC9C{Q@@-vFCpY=ad_p&KT3ns7vSFFA;kDMQcupL->iQy~d;lnc0GwD_lP@OefdzWt9Jcpb&p1Beps3%-N_|BH3 zU;@Hu3``4?5RIW-SSI$j31ipOco8uW$a)Pa?C4Av7sLE@?iWYIUF-c~imY#WPSaJ$ zmV6h+H2dS4ECI8`5x8OErM-Lea4b+yqof(I+F)xj4ch3dNfq(X6;a&yyeblLE!Y8( zGj2+&-wpC}_=?>1>6_0dWxgbhwPQz`i$e5$PfOQ{)q46v-1}m!@dTpwgpSL#)}SYa z+uUigT5&xON7Hb3f5r7imzD;P**NnhKVoyj@x;*0`c;n#Yc{T35QP7CltkQ8m{!aA zF`XZJg%L}V0pxb;X#91sNX1cDyYbgip&eknMqZ{>G`V_Rs95U7Mg%nxF;N;mp4W$X z6X{?kfLP!xEqnR-U%;95zmL&O2<;#Y$>Vy$QxQBoRW!jkd`kF|CHr8FRS`TIDQSCB zKeZr&?;q_52yCeSPQchqqw;XN+Bo#CQ)DFslAV?_(&SNgIV{ zrZBwfQ9ac){`Q5`TzXVyT{lQrU99_$idBz4IMj^`C8dVD2WwRw9oBbaAccnOk2=bu zQ<6h;=TRd;`qtSkHvey6&gf44Gtv!%TZ&-jNvYy0sbaEK#bZ?Q@%R5E&N8*jj9veB zCjNx=Yq(zVK6QcDf&5Ns7psvH>}~N;j)G`n18*bVU2UDb`f(4L3L8mIS0edNzOKv z*4f-JX=_-J*ZyYzzBPF~MWP?$9ODImTd;Dd7ul$$otSJtv~Zp&PMH zi04@C80nk`f(dhtswT7ZkE_nVmUhvi#?R?Fnq&No1#He4<2HifSFLgA5Opr{z)Cy; zN9;Ls&=5*s7{iks`W2e9Fm^|L0!A=foVbQ5HNJYm8Yj;tC8gjukT_jQ*A+V9W*V|? zCIs{ZJNo|K=vh*hik?q&J9bO5w@^(Q%9H^DO%}>9=o4gk|D9UFmd(aXhS_HXN{vPR zIj_K+-JQ+{XJ-uO-^byk#&ak&f$!zLg+4(B^uz&rf(nLrxytEb?SLW*rb-W^rFE$t zU0YMVn>7Mx%;j>aCh}&LY#{WwMAWEp&?2ZOeXP zTPu@O16P7&_O0Z!V17B(d`B+#b0AatUha2OO$?DG3z%x|?5So8?uz4~$q=ofwpfm5 zhm&EO3u^iiN1h@YCW-KA4CpW*mNa{z@_ZbZwsnX8=-KfGpzMP<)5HI?vgg*v0k!Ey zZEp0&QSNLU8G4+}L-(XLmv;m|`6}K6O|hXj1C`6lROutP@e>YN_Jo=Xo#a+Dm=Ith zOE%?qA|%AGx&id>QqEg~ew$PA=U>1#Rwr0ZD1ryH{Z{+JZpOChAc4!&-BRlcbBfnbxNB8af^14V#1C(}|NJu2N z6eM02bMe>9O&c>nn=D4sS{bU`dbekjkYG7Yn{RL;wXMz}a%}iP)0wtW{axP81&Qc9 zwHCN0I6988_sH2{(|u}`(?#8+p(>KbT^@c6TujUXA_tzQrBt_p2Vyg@uiv2DL|~a- zOGD3QLiIuU(4Xj|`FT<{Rn^P#i)$tBuZmnt1T=##( zfeGS&gTrx!|4Cf3IUE;hPjcjTwnpk7o6Y3sbb-M;U4w*gj8x!8=tQJ~i>GHj;dvDH zJ4sQMTl5%*Wly*eNR_YtLC?KY%+S+v)H6E#@(b+jopF*KMNV+_&uYk~X&xcDRO3JD zxJ)92>mPGygJ;+WJ)-p-^tPJ^y%P%}N;4^ytMsH4S$1Td-OFc{zP9rG|KfVGuSMSAPWqdA8`HQfC9 zx6y~w^!moelFm@V3@k{Bp;iusO^22@_47B8w;T%paVVDdNxW#Aj2V|WTBmkEEMPWm zlG@G>d8+i5s;j|8p|I$S8P=ij3w0>$g3ecm!cXZ}>qOXHCn-NotACP$+vRe}|1L8+QuG0nx++rt zQS2gh@O$#>2R}m&a}i>yxv$qh#I2QRtaXVXPto(SWYWvh@EfnlO+KeUwkprvKVxR) zGXq#!a?$mc-ko!bscd=%l?h${(kn8LM{;-EJ@P=r-{?fbt6SBUUas=BncNKB--uAy zqn>b%OjR|{g{er$n&-mg5FY{m?L0q6MQYN{V=)}H7ejiu{)huzH*vW290%5-6G6lS zw#~!WdK27?`gV>p(J6@fJ{YNg8Vyj-jug*t4(j<#>_?+hGJM_4=`VV?7>`EQT3n0- zy&e|RgW|*5NF?cOa$q+99#(|z--=k?>2clX5zy094EldtPhZYJ#J8Dk#v`GpS$dozxLcW;8QWP3oa)Qor=Ale){F#fkVn z-T%B^pNT+K=<0{%djEN_z~zD;6U^?*^*8ixmHFbdcyqq9CEqn+b9qw7)6fDW2@aa) z>t2t*16di(A7^a*_zY_yrzz-;)IUnyKH|W-5R z+IxCEcLOh(i;N)z`&@JKFKiMF$NJ0kikI}ghyDS!fsxRTPuoKufnyHNAt@P9-@lTU zP0?JUnlvd7rnRV|)jz7UCo$DL_qUN3cw65at=~-*o^l?3>M1UyxwWQJ;z!Q^Qgk9h zFKM3u(Dypys-|fN@u%zGjUV0gNi1gH0cL$$-B9FB^jCd5P1zKYZrHR3(+Ne-hY|{f zIl=Pyf{5L9358CMea(8w^ZEhuGloY>I7+uiGp|cKNF}1KtEqJwr@O64);AkjbZbRS z8P$J!CiSKjEXFjj3p;VgwSw_!Uj3t6@O;>jIOVE!{#d_;++5Awk}%gt)auYd8X6z@}AANCP2OA^&Bg1_|QC7G~`auk|bpp&pxKZ|90e4CUhhU zQ~hnz^U2~Lo97WIq?{)4F_rXu104D_j+is?+wmSuW1AOJ8t6un%>`$$BTqsxYLy6 zTsU&?g83Uy5ljeWsdpj?_G;%SFP#;;!fkmKyZYxf^vGzX|M<;%4Ey>|=j_>(QxF7ct5VyS5dVv?5DZTvtWd_AcRklYd+>I+>2_IN=$Vbe~t92Z>Obe+Sa`{e5* zKS4I66IVu!qMuW0Tt|-*znmTW&E?%R15&9wr^{@`0@aHBW-Ioat=ONaTCv}3#eQi8 zBFFmPk*|g>rK_ak!+4t&e^nd2alfkg0NxZu75A&kzpE;rK@)+}05Vby;GM`I*tO>; zDsxlc$3+3YHY3vP4~IM5PS1aW4E-l^_B1xFmG1tfWTSfgm#PZV>AzH!kbb8lq~Cuj zCEH!EAAC_ppZ>m-N_bVZzi{O7mi``W^|$n_S#8zl!MxEj>hp0wH}tY5FlHz{dzXq# z^9Ep1C$a=T2+lC8!XyxU!Bb|%Rf3Y3Elz#cP zro5t>^2WC`<&A&V6xEP_tsTv2`*T`;Nw-^_hgWiZ&Fg)vcCd-!Szf`K{R0fj zK%~+@YMCW$8Rus~z^YmDgd~M`@_x=Ajiv8?74O*Ja7K={Ff;{?BF&p&-1>VNN>kTC z02Uji^MIVQvlJV}1mRu#Y~D(ys)JPEw3wC_aevtBqJWcKZ2UD1jW5m{ml)m;glmAw z0*7(E3W}Jlk^Mv5`TRvfx8DaAKg#j{0;rt@fs+b8^!u_{55z=izsQR&)-?^ z(pzJTjqQJv+Kr$%Ns|(r<7-(DFGW0pble>3Nom;s8^Z8^tUp1 zAw{><79gHXJiLqd>@gppHu7k9Xz zE2k>xxHZZ*O!;Ks3=L=jqT}$Xvp*TSlkWFy28B8{;8MlY_zIzFS!(!88T34ib3~iP zVG13_aGYoS0r$@qXY&Pux=8T~AEMgPz+^VozQ%Nrvj`4^HZ`p~XM>x=2f5WE=*vr1 z4zuJ$nek@?xH-H_Wd=QxE%8-V){NZH2FK}w*iPdTvJX#6($b8%s!SPo7^(d#xYz0| zGdxlf6Ym6U7Q3RiQqdU4)D1Q3&@rm377mQeNZLQBuOB#U(_uqA>!;*E^IW77=E}*V zGao_0eeD7sCazn0`u`1Jex;NVWL_rQOSEs=jd}HT7UIHHH2P;)xsYqR_RX94v!=y97zlBO7#7u_9ILppgy%TQ!Yqt ze&>f~@Bb5}Dp7Yj?zM=TD!n*;-AvT5WLAFaS=r>M@~k|Fkk{xq_0 ze7vX1qqXr6H^%#oW>(F_aC1iN5Nr!Hr*Qa*))zozh%u5xAI==v92=pS+B-v8vR*ty ziv8U45*(V-&bHVaEG<}-46UvCWKI1ti(iRn4!A6yMkVj9SpP$emcR5YhOH0Siy=h+ znM`)2p|&9V5PmmALv4Y{YgZa-E#Kg^n;L4Td$`!miE~6l&EmJ48fuE&9vj$q3CYgE z#9e65$>W*DV3smgp{MZ}-xSv!S1Ln$tNPXbR#lb4uxzsN&|lPQJ8rab*UHsOuIb`$ zE~_m2FTwmNe#DA?3DFPNXyWU2$CSD(!E{(WOE)AMvlVpp7BOfrz8jyn7 z(P{qXYGww%X$J&2OAzinmd*M!pX&zLe_gU5yD?^O`&Y5)E?dPX#13+XiyqVSth(00 zQzbT{=!m>(170xto$TUIh=!19ZEqOV20mAwpPk1U0DjEr$iPUkD^l#vElKy>Bjz00 zL`yOvMer56@f&~+2u!PHb=HRY3Ynts#|Ho>?oK&+b1R>AN{+buorj%WK+-xRxD|Py zGw`WMoy)m1au=3ea9<+FTc7`ZQ^McLe7R6>!Q{)z1mr7{!eDKYNG1Js;B&c)PSr!FLSAy zfs12^y=Z!(x9!}>U21iGDFf+Er0q&5PqJB55eT*ar?L0IYgrMJxvCY|$e+94ceLHTlXeA{|jz) zJhJx1!p|lQPoC{lM_sE~E_HgbS9T$zNw>Byl{_T8=>lsr$g*fk3`&m?17#Nzu7Z$DkDl(cbm*G<#rHJ zjKp<+4tI~*a1?sTvm-b99WM7DK4nEvqgL zo)?|*GbwG=h0IF&I{c5st!`~0JCT#l&w0D!5o!=TubgPy#u9P0GyIV^ymJpDL<--b zy1igs4%M^{+GDVmgFazR+oZ6f|UhtbOl=s(x0I>|W>}qjOiOY=-p0erLm+y|AZj z*vGWk%JF+P?Cl}-eJQ)^YQS`043HIhnsoRf1l8=FXVB06zEht=Gy{oA7Vl;v7vvEG1xgl zVh#*9PlzS`?tInPWb+^LNmSPN#Yb0z8sekZ@b~{a|7xB~am@Z*iX-@6DUQ2&?&tXh z&vu>`p0{|8^EiH+;z;MYkY_y4G@eqPWjyP6w(va4b2WKg%<~rE<2>Cwu4hslSv(VY zZsb|QQ_HiS=YF1_@;uG6ljjJ}M?8*aDUauTo^d?a^33L`=2^pYKhI-4+jw5)d7a1L zIl+@kn?~_W=DC^YZl3S+Jj=6>$KXlcPCIz6=9$G)%Cm&0mS-K$gFHXy`7O_Gp8Y&; z@qED3&6B<(#W9R$0#6~&tvo@Vbv!@hd6MV1JZ(Jh@;IJPab)m};+euTi)RT>i02-j zhk3U0yv%ci=L4Q@o>cnsd>;At0(~j}0x}+7``30$zGM7<C+q+ zEtuwb{aAia%rCE+=J;Hn=JZPH@)e>5=w6=0_ zXhrQ>?-LQ7Xw7O$-fRW852rfP-b&dS=F%4IoMN^P3=>&jNFzIr0{TueRB>dV^7MWM>CclPC^?H!Za%H=Co zeIt%^ym#>mZP^lU&5Drs_Db)f#gcpF5{E;5TfL}GeW_ZZ)d0({@p`qIJ8M>~uJKmZ z)~=}aj$Yzjv3RjoTe-x$x>|an7IrJY8#Wa<#+{$%xR3XUp?Qvnc_&_w z=XjF$7~T&z7C1&We3t=OT+OPKH!DyUpcJrhY2DQmZ(kac@X|W-J9#c%&6xAq526EQh0jy4NnfqdMdrQ!v(hMQ-lI%UFw(yfToW1KfoQgGQ# zOIJu93k&81%5S=U#>|@?j+y@a=?WIO$zPOzgMVSE|EA*W=gbz^*?E%PmE?Ivj(66g zx)n9vKxOTnOKa-9i)xm5L)Dd_i^}DT?v(e6ASh1CvGeRx-f9!~I2O(+b_DWoo~g>t zFSuEXT$q2;jABVQCm_Ei=rFNbougP`w>q!%(PZ!FI-#T&t~85WxO~yln(@mt!6|l; zcHWXlNe>>bqIB;!lKH$^U8VPOt0?aZg+*Dsl{Ma_HQptaw`+HJm#+<4 z)>c+k)(SS4b}w16SX*9M!#IH{y<@-tw`)s5A0%Z4s`!$a-yq{fOU90ONbN`0UH*-F zNL^KFyqTYiD{o)4_)fxSMU>xmBMasf73I%X;|JJLrPVA}l~_?XzH&|FV%9;`n8nMN zjPs5kKYnZ~A@5YLIbp_E2~G%^U$vT*6f|a0?Hyysrq))5wAvbPZx#|~Y%23Eb=iu= z>;yfftXx!ESGjQ6l7)3E7A{^C$oDY$@67dMlpvAmgJ@ zh;(<|n7$c|O_g3*IL^CDdS&U#Mf8Imp-%OTH3%TfWtHOvA&wEqIM&wJ26PqkK?c5Z zSzTqS8J$^NSsStvNJm&PBo>b~dwK_`&ykghnuzBoDJwz|EC4tHz9w^-rfu&3m(~UbOU1 z?OQXXPUKm)!pu?v-GMuc%w|-{ik!E#+86nBi|NWqSFI z(-(eQQC26JeekbV`RiGEnH?iWeA^`X#`ur8oE`7-WlQLf@v^u_j8*f;>TUHJKi)fH z@(2)&ca=F+m;`6d6sykWP>r=mN~JX`Vh@knZdtl&p?{-@%W3Y_6DMD;)zzB8%O|U# ztRE9FLY4om344-R0XvbI#9A&@%R_w@j8$#bUUTuuoTcZ_#-Ai&;h#OBYb)!tWrBH_ z$Wk41OCKZYWsQZZ6)v!1{XguzX?Gh(k~TV@`77$cRugO!AVtY)Tedtck(8~sweX6R z?LL%-2LwS976`BbP@=};|9+n*mRza|3*}|}-rJgK6IGQN85tQ{Mn-0`x3CkUbM2K# zoXJyAwtxlJZU+7AHnKgMa6n;BrTb}l2vHxJ+kiJ6!u`SMeQyjxL$UTx{7J=FdwO*5 z?xX(nyU}EN@Zqv2`g)~t{lz`JZ>+C0`W^k-xj4}umzQ1pLCF3v0Xy;i$}cN9RyNQS z93thORujyXDtT&Y>60Qww}@5HQ(_*}gVy2Kk^*@7VtaRg|HVsq72DrEIoRDkc=d8O zssx8)LR$tix|&{IO-HBAMz4E5Dn9-0#s0y;KldEKN^CHe#M}Zb(z;+H7(Xk*KTc(r zx^vKW3fd+NS0cJF9=a>F#d>+28B|KN?`ennnlF@ef&3>Uu^CNewDsbmGkLpSyzdOB zVn&27`R=|rrs8Kd8|@9(X#4~sYYCXAJ1Nr~!!d%|x9`?6;aq}4t&&>dTYpt+=pSX! zV|Dv=-^{O<3I;fqq%2V?(5%j>oAymQ;t#?tZR!|Cr;Xft!;tnEdVMLE#? zpQFi%nvudsUbyfhNe($BjU0L)wc@9=0F%z374;yiTZvrwXmS-bPB4w5YKUo--C>*2 zU#CMYd-7voVDAgyPMR`(&~BKGQWxGVr^rS0H93!<175FfL^in~>)#i<@F@2l%het9kHwoelJtq#|G%uag@qm*3{%P|v8$aky$n*w%KFBrc1*<52 z#^wDp3QAj2{cUbzLDUq5*VPHi!#@G*Yu17E(8Su^LLa00-=VcOj6m-vHa&0O~j zHPrM#j3WptuucUx(9n!`5GJ;NnKVxEsWuL^*)P5+Zi{-vPp}f(7Nc%0Ll$)o-v`wa zjL@Aq2#parkHqMn-xhQfQkaT&&$q(st@SOjVe$^<3-h7Wkp+a=t-)~jhK#Yt7R%kA zOMC5NxqoxIR~_n<)Q20dU%yfJtUSurlEx!aYF^IhU4POg7dzXkaEAWMv;wBSOWoL> z$bT<5>kj2eU-R%nEdv((X;(TD%D@zLib&w38HyvQf!``JIp+bn%gbfZQ~Xd^FzjAm z>)#A(_Tb`=M*bXX5#FUKg+MjsCa3PX;CXI;tP|(x#2-j#DCp7ExZ7KT+T-pYhu9e5 zRoIcs^|6Tem2;2D;pCUC|QP)e^~Ln%-P+Ot{h#1V3vQf-Tca`Ew* z2&`mhI>p+eM0WT?d%|D$&!{sS&BI%t-D=-z-TJRvpWbTTTDx`Y%^JD#)~#E&p=qps zrP{5(VxEX%uQxa67qdcJb6^P!3Mp;*tOikNi?Ha9v7Flw?Vsp4HLuaWS}dq{eK|mc zzbPS9p$5i%08ONGtCm8BV65I#=E_9oP{^jj?k2ABdz+E+x=yF;#?c z49AJax2+*GJ|ZfD0fEzt$R3Wz@#?$^=df9s)0#3`0v;rtMZisPRw5cvv1R@@x&I@CZU zhSr+z)+F*HnEFDEwfN55!TMe(`XnR&mo>YvPyGv31wz>49?*nY>6}6xt|OOV`o7nL zQc&&>ln%l88NIP^okT`7F5-}!0W98Z5HV}90@8udPWqXMNzk7E5&W!d5{TCBJNZ*u zktWUQF#==4h!YVmB|7;Rb0S7)Gs-hYZN-zIr&TX2s1ke|5E`!(iyVV6UL}Mly^G7~ zhb9z&4W7ixAWuLM6Wh8@ijAKib4Zs8Jl^4qr?6|fpN_8zbk-dm9}sC)f4zV?w27eW zm%6^Z?R}V#P|kIHhIai~Z3{xQ`UuLs zT^Guwnov7@`1TF@Vhc}X56Qr3Hu^&_R{un~6Q2kmrxB1yTk{k6RfW0jDWA8b}Vg{UTd>LQoRI<(r6=O^;H8a2=`7ID^f80 z72Q(6l`xW+AS`XbAyb<#zk-k4@q zrd){suEWy^6i(zP?Tb#}9AOr6h9Idfk=-*rZC*@}!6O<8qsz8P=Fz}H5lWG&%1e9TLmr6Ot>uM_h0+{eb+RKt!~n9r~f)-|4~C z1>tLG{l4Fy*WMhsKOT>+@7&v*-IZ%Ca5M0psv@K75iA$-lMw?SXi`{D#I=?SxMT=xBfY<&(VwodM~(lPx)^rZT;+)i5cvm2v*ETm(~+ofLp}kI^R@D2NHs+jRHZf%v>b}N?X{x(SeKap8 zz^E7wdC|<(wV0Ju6|17m4KB?CsLDT)ch!~gK|torg95{LfdLk8gYGu{kPEoE5(?Lc z_R$e5=Crw(+Ur0F>#&cbw?*41^Jmx4n87mTGFRLBvOPeKxqV4!A{nY}E7CbRX?8*v z`p9%2p7~18X}_l_oR&ZLb0bNxXOswOw`XUtlfQ(CIn8qCP%6H$9$qL=;32O=2Zb8? z%}Sc%YCU2A0g1JR}ACFC-b(Abh*m>7FA9rGW{g84vx7j(Pf~IT^r(7Vm~@pyQup)}iUU$Sy<5 zwgbG>^kJ!xfVS5WvX=v_Xx`ZXh}&sf`l}bWN9aA?NIfRb+y8*ksdQlkgwbnvhkrcmT5*xF4IJ%dzt>VMEP9*>dU$M*NRa8 z;-@p2>A9I2xR|Aai(Cf-~4JOT{3Mi)hlwEDd2reisdqG{1y~u+kzV*t4VY5U3~2&kE?5B(SJKA{3M`ButYz z&zJNDN`2=5ptiayGJ^*~6J6R%Y`B9% zB_9!I&$7yt>4DWRsi-sCehdK^RU5#rQ11;&XL zxHL|OE|E@_z*hi*uc7ZVC_XZ!1Y?zaCMG3RkU3_p3(ZrAicu_p378V*Fo1l+2FMKh zhBd%=;X(4>xo}~N)Zv5o)V7nvHaMvQ1voyT5v(IqQ5;r_rRbUgD$rvdB zlHR&66EuS~o*{+FC40S=FFh#2fgqufle;a$Awym&s%c~CwQO_`O-FrqY~Eg7duDZCVfv-zt5pPmO04s4_mfaory2eTk5X1}09El-x5^sxQtL%Oa)5Rvk?mjR z@PPQhQ$%-#BM^&cFs>&e{`1OcNLWD7du$|6+~=$@xLP(#Xm-|*!;GM#T=I2sY}6fs z&8YZ(@qD*hyI;Zxz=!Ck#tg=z5mu4=!~V428JzSkx!FWT8Y$1f8F7-469);pb=^>S zR@d30KT*o6#VMRE1zVIk9b)z;z{p|6N_ZB+pi#l3@Zz6j)Ik#8rjDUQI>zSQE&yTK zR6hv3%HfPV4ygW{8ty5SUBi`rP1zC(_T56VQa`40Kq zwNa2zN}uSGI0N(!@GtsKn~<`h6;Buw{oC}ROSo4HDr+{u%gWp-KMQ#yon{m=A0qV^ zJkEfY(?Gw!o;zm=g~W4^dA*VqfenMF((lHj9NXXw0j?3lAnu6-m@yUrP$+^Ntt6I) z7wbS+V`mQHP*--rsyfORF*1fE?^L1+MHW^7BN6wg0^xv(0lqGjnP_@r( z#(_^Xg_NdG8&ct$%7U$-B6gAD2~4DA8gK7{BIKMmc}GXdoJ?@fqjA0Y~F7ODXN&cp)3*${A1=(TZ zDl$;NtmaIGc>F;@a9|C}ksPd`)SMhn(No0c@wqKLB!v_=?-y{CPm~5&Q5L0L2+n6< z8mCe~EilHU>xn1O!%bO+k!kvxOlig3d0<2Ftx1kabDpd+Jd(^*YAp<$aXwBmLh_z^ z!ZzkwM5iAbMzC0g<`}R3Wp%wM8h`0rTz-Xxu(}crE%C3b(R4lh z2ntWbu~hvm1>u`n%3%VM6EK)AncGraGarZGG6Jv%HLNg{OaYaAr26WX5YH;sd|zT^ zLrG;PO3Ja2wfXyit!d8@f*|8b=eqh&4E%o;Ofo$wDw-;YDq41&(G#YpK?IB#j2^L* z7+EVMx<-?48vM$MMTxOkKhQsj01N+&nY1^ilnhWq@@J3%VA6z|B#Nxd4VwuXVg|s9 zD8>I`0p3Be`@uH-gvD&v(kM5>3k!r{W8P>qUF$rMXmXBKC{@`&!x0mCRFnl#M359^ z#Z^<5L=c=1k~i!GllUAts^oIJ~fELPhcyJR9D1V!wkH{LrE%! zYK((RRyGHUAek<}*Z?{bETo6N!zQdGhbojM~rhLxg-MF=Q-Vu3By=CHbv z;STRYHd~8Y3UPT&%V#}#kJTBFPMPHxg)n?SW3sNaz-yuno}nnPkUokeyB;)3AgfTo zXg{?+yQyh&k)FUmye3F*i}h;ChSzD?Fu2{$ki=-}lnbFqB)Io1p+|qWrKjAHzAu@5 zPjRY|FnHxNPetqUb8g6Vjf!JI4Qr}KwqypQC|d^eTD1ioJ4S^vU?-#LFqvh~g_oBN zAkDTduM3cVdSSy1PFC&Q=_n8Drt6}=-Slt$k@Dd>q#w6PB|;h+d2p;=E{K_EaqU@D!j`8T2M;(?M6PiR_t=**G;sf3cW8)(Ze-`MSwLVKz4ue(@9p*W$fiY^OFJT=8CSwNwp@H!_)*YvF zq;gJmp6^zsAyu*A(Tnzrz@o1MI!S}>qXmtgj|Vk|^ojb{7_i#7?;hzj2@_LJ?ElrKIMP)g zS}hy`kwi(s)lQyrlq-Hny&GcVB9=%^j=LNcFH;C*KIa>;f@;4YAkKh*HXV%y0w^!h zz|_5et!xv_SyC-fRFOy+)3UHa_8;@bxNe_Ifngoqs{iY6lg3GxIs{ZlvUNCoa`1)t zF##`if2GG*D-dfSx?Fz>M#qyAQR2xf%zZlJJ`O?@&hZaL1AKdB)KFuhDn>jC{+wxi z6S0H>nxPjkBHmO&sI`?~(MZ3Ne-QdCU7=hxq&yjtBae-LXOs5x(US|N;K6TLkuDkD zjs4(G2%nLYql1IibOgkv)}WT#+6YU@mOzTb9u$z$PqTu#9MX`Wbb}PE>65g?{sf(# z&_Q~nkN}4pMnzF#IJ6NW!WA3w2Q`xoIU9W2c*A)q;VK_VNqghUh3^ay5R;Xk91z}? z;jkd(%T#Elw4$ZP6#E7Pojc1+ii+!4ja>t{WFWs8+ zZ?rq+4wyTY(U|*&bWBnTe*}*VPHTANX81vnLk}P`L4x&VS@x_b#ZQE3#RMo9T76lr zYei94f329L<#q8=1Ayh~;R6K-4<^B3uFFL^eGs`9&ZAm#7PLmKh<#SgWxWEQ=7kwH zV@p;CY^rBJ5iB13y;V}K z4qmJ#G*^`HTuYVPb7Z1NRxaV*S*fAWjY>^Nh3gf;uQNA9fz0&7g6FiH@%J5K?QU1) zD|g%ulkXbpR2yb#_%T3H_OIA_mibP+EP1`uM70=@LB6WCmltlIb-S5U22SFk^lhEs z2XkKW&D?y#w@g;euv?8P)6JbPwb^Tvmn=x-Gmuivi_SIJz+T=*pbXn)S{lC6Cs{1O z7TW<%edl5F3_r<9GLt+=DJaN^zX&M#A_MY#t#lHq^!idtM}@1Cj)NS<%FeZ`NBM}$ ziNZ+0Go`9LEQk8&8Lm8<$4^qH`tW0btZ&Jg<;-{DXd^`e>rvMVP?Dk|GCU`UiF#1& zEA~mK>Ruk-m5)`-0BHe0+0r?V_LrBR0CRX9Xg?vYAsN!nSdLlCIQ|^P(}H?}ksd^( z%K=u3t4(4G4PBw64?f>19%+we6mh1K&tkk4odgIh3MUlWsqQ&;l=QgbAZO;-Bk-0) zZFE!sX4Hx&_= zjotrnaa=ZPv3QvkD<&!Q1j&)GVlWB_3^FbcghfLW@w&Om3OgnA0vWL3Lwb4=NTDRa zhU&EIm~zKfo}!f*om?L4Qo};`z^m0!$lv$~l-`p1!gj(g|KYrF%xh;-Zm}2XcjKes zNUqtE6CIAhTAJgi%5!}M3%2h^IMbV+7j?usJbAgyCV3N<2}2p@QNV@{!=a4$f^!S# zcOZ`wXY5K=l*03}Z+ksphd37wd5}{MY-Sx>Cg0okX0KXd5f~hLNAGaw1>!ZT_J-rV#fj1Y^8D!&yxc*xrVVvn{j1fLTTD|^LSfkDC)|A z;7KX7bIxPz(<~)pBPDy6xQC0whLtK-(Zvsam6<_Gr-#0+4ZdZbHZ~-oL=MywE zYx>gbi8p1#615}>c>#tK9YbTi9OulcqoQ;V3su%BhFCb>X{DM*>4Q24`VADMt%_El zZs+8utBEeUHeFPan|Ta-TvLV#-dI?eU=@`#?ZaVDs3{*f{E`RGjV;w!TxPT%w2E`FB?o9e_6zX#I zv_6;%yam@&0(&4mX4jqw2g+$;AGy0>DQ9GHe@(9;%S2OX{4WB7R!VxNx?8e7joSEzGY>^3jKA>GIxP}|g{wrRqfE$D5Mr!V>!OVK8bnS; zQYV>(Vs*Gm!Crmyz`q8ZCdqEsOd7u`8*x-@gKXlhxdB8HAs_~o9=;3zl$gCc8Vov@*uM!9+n~En)_5tZf;~h8@KgU%MZ3Sw zY+r1j!T@=qDR~^vuG#5Ip@ffD#WR;-2^TL#G<@({t?79mr%ebCA`nV=&L!xSkS|fp z*t?~|;XAPsYcF+=lmgPNilI|_p5XpUHlsyxmga+ZvXx;0s#-kJ5H8Yd=!1}<+qo!k zAj(K?&y^0rFFfy!gPC!Ckax&gAq{WNQyw$%OnQ}|X^jRf1w+9qCv@hY&>d4Gy5VT8%-Ek|g%gmTV1G^adG z#J5?hFoCHSG#h6a4D5u7Yr-my7d#RElv5yr?}<>c+@;IduBBfBXs#C9;AGVFIEsg; z;#+2@U`a5E{riFj)#9mw-tvA3F~x zF<((-lhrYgtD^aLUrkNL#(x(@Y3MBI53 zgYc7_-18K3LQ$59jnv~oA9EfAH;Hjz99_1iy)=)}+(c_UkinXTO*?rt>FtewQp-k! z%^+dZ$8&IP688XQ4wDwKDGbWehiD4^mhum{S%tsc`6NtG9UFD+3?laA(96H^3(o?e zbD%R~GBzm1x{4~CWkEFkVwmOzHXT2A{(L|+07i7M0ty>T`ExWiYNwsBQHv7yj* zV1-CPP>P|nNGdeFJDiR=@%tHo*h!YL0b0i^Hp5p+YQB-?4fWV8zc^dH7!C;0ANlDQ zw&`q|A>=5qKo)lp{Dpcyaq=1{xyF(ynF6sQJMLwUR0_#BvDKmoObR5-t`HMUo&^rv?O45J)WHACauQ1$Q)@j@^#BC%K*Sws$ptnbUf~d2 z^b%`kIGax4+PEN7nzz;!!U7^2gVE@1VWX|XX>nzg`m136SMQm~Nj*2otPIB*b2F)d2gN`esNMCpQ=ah`9FI z;$zyIFkEDT{(8EST8p+!Dl(^0Ny^?iG2YNQ;v6(=Yy`n|#N6bQCoQHRBjR&_5D-IL zD!82LyhBo$s|ljArt957Ue55 zRx>GPsOPyPjz${)fgC6*?vF%2Dq^IyWHb?X)&$jkl9sS{f!xR5SUm;}<)z#uj)1xWVfZOO(aPPBlog`^ z;;xfvKP{9iW~&0#U&Zc10oiB=%gD_E-Lb~ zrN+v-K9$Mhg;LdmzM3Bv|4}jE4wT51vldZ_aR}CfcmW1JgaiU212TXW2+H+IQc={2 z3^O)?GHTcDP2|WTsT7GI?6FkOOqnh*>j8J0Ldv4e1OS9p7cRhD$;kM?p2qWS&O0}X z@zn}NbC%I@V0qt{@4uo1OE`MhOF|*+6&7ZCizkC13&&RQGrgS2bp$ONoCN6FS5R~h zVaGAbVjXKM{5ZPRagVy*ioHQm{LT8Z}YSl{Q6nYM#*o1Gx&4?DK&?UjqP?J<{2*VZ> zZEi(Lqhj?(W!9hpbo4zBAZf{8O}L~S@RNmQ|Af5|`SpaN)usD#&Vn`V#dR}U_Suu= zWsG2B`H{$Ff^&v()hjr)&9v@d_YGI#RUbxSGvbA@)etwV!~Eh zt#{oWT%Ev?E@f}q`-Cf5+urH|R#H_jonx2mrKY%f2^E>rnmlMBx+X7_AMz>KT1TSr zhkg)gIVY8pysV>!cGVrj?T0)#LJhT2!h7o*c4($4$kG zyO0yx>>Spc3=%6#Vp7aVN1xeMk7`+hg&q2eTp>d^H?xFeLY$ggj)6NC^OFSwBJND3 z8G~$G40{}-aC9uki1>=UXLPW8-&p|l^MPk`Sz(L--bMKFJZSa7v+Xdv091qpup2U& zUtd84U2Y_7KtXYZ2e1^cyeO2`#3|-@gOIe`qr;U=oP!`Ym*O>k zUVh-;U0jBH!XMAE*w9${h1a)AJKWxc2)-a25R@2KT{Eba&G#16^VuXCMornthBMfb zN+8C(wxoun$2b}4si!pXCUE5^_ch+;01Z1=U*i{YcdrneySmQ}p}x z`pQo4`08vIL$m{~f5wZ}OYffyvHpRmgOuJKa&>fnf4B6wWzW~hp826O9zrfaL&ATI znd6}JYB1_dA9o}I&k}sxhb$q*K&m|J3^C`V!NJ#OkGJ)Q=I#Bk4TESeM)&SM>QA@N zd)#g(g`bahPx{k`!;@`v2B#ERU-0_6i>pFZvIzD0=<2M15|_Y2=ND$xB&wkqatWRs}8 z+TS%e8aD}psgEzYPg-&*fSvCam3p|IxvMwK;LEGwq;qN%!l5zGpz??il#vVSCdiRWVf^^+*7g~QtRJFRRP^rYej5l*ToG_o#H@fb&*Vs?tkoKu7qqt2;F#XX5b23inRUR>)fv~MAs$i{&x^^_ zVJ%%sRp~awZn3##YQdl3@lrQ#&CZCqNj<$i2}CJb6koWiGwN0K4#T9Lz1z?QJN{QkxwCJZ&8tHA6Hu>#EONGuEpS;V=x~n0%kOkKh7l$`b4swG+hfvU`4xRSB9t?ct9-``v#Y~g9(NxYB^F~;!e3UKF zsyDR-TI=SvK5-Oum+?Tas4Z+L70<5fkeCHhc=qKcJ(zg+)vO|o2D$ZL?4te7Sx*X=8DPt-I@6OSC{rjNWQL>m9>aub%AaF*>OcxQ{Esdt+=B zAu%}mOrgt0zpU_5RV0TXCFQ|gP{{FP&-G0 zN@h=Z&HQ3#K2J!oPFfkCn@WO(2BXsJd;%nJTpTZPz;U7ozz4@Xr|<$_TzOqDKi1#a z8;r|&gxG>gIu!aA!{A-PI#-Bb>!y)rIL05NpfnYHF^dHW!aGS$(s#i;8RDS~1j@m0 zUyXTb>2?4qsLd}4GikZcf;Na&YA=kH3{?Q8OY0W(v*fGr5jch4VRveP&t$IfS!FKt zPFo9K0SMkvAgC&li^*A38s1kUi||y5i1rE90Iq=xuJwSLkRZWDJ@1I0{m?8?VG#_l z0yAi4w!u|O+4g_~Is7Ojqq}iYoCk*9<7Ogsl>%MiB%I>Y;l`O}m&1biB~u7!#%!m4 zB6y~Golr>P7&8juOalkgcv3(m`q*n&-%)KeZsNb|1rJ&SRwTA)ZZDaf7+{q`Uy@H~ zoa+dLptyzJogk(%>5c8k44y(Dx5dh!l50fvl=H+ha~)At1XdDiAi^|wdXp+!mm!mq zWuh$5W9+oGAnPXyX$)n8SLe}sR7Z$-ssfS0|JczKZp58_{(A9w=KR-ztf#7-SF?a) zq!h=(cl5!1!=a=$Tw#znImo9ZHytQBf}_0NA)%8DHV^8k9W#(gl#!8&D%0lfL0QQs z^C$l5Y?46_y)}yGEm4@W@l#qrjG1*;os{Xo2tz3|G@cBB13WOngq@}Ry>A&*h)^oV zOh(5MP&{iQ=~5u5a48rg!(1S)7dR%-td5KF9HpS>E0$p-lX~za=W<7A>twVXGht;? z3UbacyIF?1f1h?Gu0FeAcsD}ES^6`aK^26otRm|4W~L047ZcSGwPsc>)r_J|LX+ti z=uB0a8S`VUln0ooGE5r`VMOWWSGE~R)AzV~t{3e=3R2JyIL96sPL|VMwJtKBz5xp8P^o2^{AtToII4kf)Dw?A z%mNwgNfl}pxv=0l3tBW2%Ybv;KJU=7DiOR^v$hFy(CBE2%p?myZbg-&#f>@oHXJwS zddnxS)^NQ9IvaW=&Z6Q3ttq)0;-FLA%2O@H;yf@`PSilq5@kIVOUhN9`uR>B!|tP?V>-vS3(2C@LUI-**|_l8f85%ehyS)MY8isYB3x1#Kc7iqC9gT4A8Ce@>50Q^el zY1M-IS#>x4^?A(zP6oFM4b6az)%Qs&1mckjIX8^94>fpatmGovOa9S2DddYK3;Rc$ z*)`z|qWk^?;W|N#k=poA3NU(WpW~sxQbxHubO`To29S>GS8+x_vY|gglgP@T-4#q` zqz0iTGrtA26BFvJ8%0f{Q|f0r8)zlpfyoW$Yzr_5bry&;!L?kqq>jt2Cmc?46aNL+ za!#0I%b}IARHvrgL>Sn>ZF33>J^>2=E=947sOEQ6BTn$YO=HS_CP0c?9)`X5S*!?3 zz@MpMWZ%wY06L$HdK2ysI`81TW;SrG+s;0R` z_GANub(d#u)Dst^oT-px;gfq4D8h3=^iS*bh~moKZqE0Q4PdxE!b1WJyAG6 zp?7gP#c4EWgV8Y#sm;uu3NuYfIws%(Nzmd zi;W=P%)%hyIPVPIg4m(C*3?GsCct4?3^>vHmGlQN&1-IU=)@=k=$kflDc`b{FkUFH zsQ^v}^~73HF#&Et?LE{6WH5|FOqtT&L{9Pjq?-Gc>aUrcbP zG%+h0&-&a?GCG|WQm@&abOZJmqv1$O;k;w)RKmF@&+w*bUUVlYgMZq|O9xgo_Hf^& zOxlP8VG-C8edTPu+SA8Rp4JS|OYTeNC+@ZG6nn}@g%09ZSO7Zha0vqHZ`GLWZhLUm z!P4toa-a7GmsM`q=}(ZoL(2nr$&T;^z($j5Z)Wh@w>_l(W8#Gg9S(w$o#q{!uz7ll zV_;zCut@-4nFgyz7XogKjQva=<5j90O8gT>d$hl^UOewj7gFM!4BW6-yx1>(beK>^a&;?;H1_=(S7i=A z1CgI1uW4Q&?s{D;2A({%G(ow=O|xcj#!?q2a$=(&_XIa#Jts(c0E=QR<@kf$+_qMw z==Gf~rznYfXRCGG$EhFMq3e5h(~tM=rXO*lE;rV722!NqbSjVut?#i0vjxu)PV$V3 z()6r~Kts+Ba$G`lf>UiRZ6>RAIk}VeU=RE)d#d4C-YJ@law2yKyw@27O>liYy1sJ{ z>A=cnKjINdJezf;7icvV(;D%R07gX{@5=#iDitS4wuk!*C)tJi&%ocD5F&(BxD@>RCk0IVhG+ZXwCB{}=lb-^f? zIt=e4*|5~qJyGhABe@PhI4bl}LaLpV(f#F~@y^+~?9WO-NkYc9ZDnEtgn$F7+FidQ z+PtZ-;mIS|wV-qhR0aqOoY$6y;-I7yb|1b(7UI>o8?V}}g(=&sGLW4~2Ov{yJCndJA&owVbltfa7`mv`43uR^rm6Jf&qCc)DC)M>Ft^XUNDYqd6_lhM z=5>lpSuTo~k;c^<+Ui7TbZ9$WeInhaFRGT-D*78uVAuUEM29li#WaL+;sDx8)GHLvjvYH%k*sm4jD60YiKoasA~Af*gF9Y9}9 z8QBu0G*KzhRdxu9`c3(9z*1WGdSZ3ihCypA8Un_QBkkGWuu28~gTa;*{Zfy2lgp)r zs-aGr$X}|2PqXXAB>z&ar*hcnhd{Kre-b`4$dM{nyy4BQ{D+aE&mHuxe7e5&Pe}gH zK4J5ged>(;Ztt#8RT3TJ{xUDtx|mF;W5buN>fEt}RVDD>>v4r{Kjz;m1R zRZAO5FX(2i3{LDNsr+=~YBJuyb`kkRSoiKO2aC?v3|R6Bu!8?rfrzRZW^`GL{+%!G z6)pLB{l&c_AfgYKm)-cQ0=uT~i=YD_ilhkEbh;Ku8862McbID*RbDyBpzh`6Rcr8W ztA)8Spn+qO!Oy>$#FpwodQr`XT=^;t`Vam7PLEd?BwV4*k}-nc35Ker7W(;$nG!{fz_N0S|**@N)m?7PBk7^*Cu3r&n{avb0`heTSM=g6iIuw zjL}syQChmxinF^eElIyi{qkwBO#L!z94%x`uf08}_`N6FdB^AZ0p4>{%{T75)mzHv zu(nK!YMUYl!BHcJv+qKswogT5$r@i1ig78mVat#GmJ1p^oGn&uE>wMBxoMpx68ZjXJ? zSY?_hDa*}Y8Wm}uUJVAEmhsbSV|6xpQl~F4t_drLC?G6DID`HdUP{|huh&%vl(zpY zA9_;@&I|kPzw-+jOWuf!WxMEXg+WfXED;SSV|~?5NeXUDZ%Sg@vDgi`h6tN*=M~~m zqwxpsNJ5TzE#|AOr!rti&mR6`w~e^8r2r}Emhk^w7SaBkKHRhm9(lx02uv`o1K{84 z++Jv+6VNy#vq|fW9>2_I(3wI<7rb%xOg`t&gc@$S{JA05-p)z z43YWv4T^bbP^PW_99mv`mniAqPsy*go?tsFT|g-J%oD7VPL@A{UY!if`WSDL_#B?1 z)|5~6@i~6Fxzhacq~AGvK!sEjW&07Aq)+AJlG+NM8PaN=k9bU=pV)+{I2V<`7`l$E zYfSskx5J`(yH3CT*lj(igs|AhlA8NN10)Q7h}v~}q{z70%aE)_2>7|8YD>ZEN{GPc zU!lF3F@h?)RTQ^l@T)X;t#+P;p0pI^@iXU(wJ>iwSNGoi5qjL(T0)WM^G|mQ=1uf` zqHPn~n-Xo}lVjA+8~`Juc5vClUOj zB5P4+qwePsqV0d=L1O%Ci5V8#lqAC58 zBma!8yh2@XS|tyi;RcY&j5-LOrx@0uNcT&I##)Uhdvk~^Z%Z~a=%R@xb!n9pI7hdE z=HwI&uKxS_JV=X4TuNF}R7Zh3nKFWMx<_MHhv_oJ6hnwn%m&$bkFX>CXONo!ZbE7) zFZfYP4T+r~v&c&_gC%V6M#EI@m0@Z>3E{&SCBY_c_mKDX8f7{f4Q`@NoLe)N z79dKfknF*B3vCz-!xFY#mzkgle&U#doW~7_9VFzLD~EiIn^5P8ibN@KetxOWZ#Z%r zdSC=lkzsqF5xy6D2Txu+-&YiMJewbS(>0S7F014yAylKdVo5KIm1);7PC8$#j26WB zJ+o|Oj~Y(GJ#9AjEP3}9`rgrtUM#r}mp4jYEc4-tqrI2A42d6ozx#3@2s0U60E7?Q zFZTZV^2xW~9fVJfTa)4z(i#e{gkLqK5PI;-!(^A^eb=^ki^Gu~N{S1xb>Gb)j<3Gr zy6pt#iiK*)Pa|jY20y9D6;i5j^o?k895V%$QUBBP*0G^a8gJVR`<$mOT9M?X>~pi% z8u=Es!%>l%4$srr^cPj5GC^A0Hsuw}xTm49RAUp2q#^J@`b217iLD%+-fsvLrz$4c zw_3M9x52RHd0I)xx?d|&w6i8>t+kZV1N}mK3|ykcyVjzR0o-b`d${#RBiU3iUI(L; zK0e$Qsi+R4@j+seO^MKa=rB4+qVP9E!3u@G+jB~Sto@|n$6y}pjqt?RBm*5A)q1_+ zd^)|nzp()efvdB}{n6C~4oe&PcN^w3Sl8P1O&RbulOGPeCGy zi-|?SK>+M;_wtCSM3+QM;QKbd-@pF|a*V4H^9*B_L5Yy+pB8ohMDzbq5ey3-ON_X{ zCUO8Ca`v#8m@drXF*#Nj5Ag~QM9vjhaM*wafng-%pj@W3YY8!3-rS0;f=E#;D6%m2 zu}G>uPTVFQ;)Md1t9WrjAx*laC-4P zf^J#03B1)Ny2n@qsRND=0c;mO{bKy3lw3^0ewBNW16zb>0xLr`cBQU_-mWHUz=%b} z6{P?MY5QP&lkZe6|t6%@;^~IpDg!cz>*f;bxMg{Wvpz-S9aqEi) zZx|e&bOz8V4;mkEwR!QM-ymN=s*{J6*^k2T9#MR)#O?LfuYd4yFQ)zJ05D*Xzrpdv zKIKy5h#sYu(fO-(mZMfuFOsq3@}H3K*!0EXaSusJ*lWRlh6{LX%3;SEujwP*ZdXH8 z$k-8$P4UZ;$56BL)f@ul0gMQ)xfIt}yabT`4qDpjL0Mv&>gqDIRBW&%p~fWK6cO#5$;E9d7Okqk`hcJpi$X?Y6F$^6G3jZrD{3r@P z4_fSm44l6)dy$+N9nuv7kFWxAErVKV?tj1>sJTKg<`6T1fTexmIe% z0>^s~nbRSA35?9JZ{d*%aPlBWtLl8g#`}f1V+9fir&pc9V{G&7jRkfx-mu6a$wb=_ zIJR()hJ-8n6NKhvZ-XEo8aI07?Rg&m_Z_tM!;|MbFL(DFY!c+b+#tg2mVt4HlaSXU z36ky4v)Svql{K@y?Jeju(qkGDl5amB@lp#y5112l7`KJsp@EcUF6wR@6+^L%Y9IHI zXehdl0%rqYm{RnW3^~2t_xfk&(+7>s{3~~)smJ=D(Yh|WpMBQSW6YDD8ux6FeO%Be zFm7UNWTY(%$-M7I6D&9!!30lf(dE6+$KS)U%RJQZ76@2!HVM9qo}%Pv+B?!O^xZUT ztd^4*34n2sE7L$Lm2AD^2YS(-C;trz|Aa*YzT(#RPjJs&dJ=C|M6{iOl0TSYtRkU= z>9d8bTfUQN1E51}089FX9e;8YOsMgfPp&r~ezhty64PCuqmw!s2=fKk`lx$xGC4Z# zLFCm=36~I#@Bk^ZQ6`O1jr_5ZQd07hY+L{8{%AbLUPBzd#4V&3Bgy;47+^=DB>noH zM9BmYDC}Y-LwJH4bCI$qK)fx@$qQEA)bk>)fkDzxiz#BR=9c*eqHAXUxAPKAah83A z$555u>%P3=qVv-Tng+sk(I4{Hy(maDIf)@Mc`><+?u81s_lN41YRI&0UI|L$=mOMx z(AbL2(BS35o+?@ohG4nd**z>FCS-&*#9>w8i=g0c4dz$LnlQXe&3l0#M{Coktb~ek zd1LD~!vl$MhtG%cSO3kM`l|soCQHtRQxwYMP$#u;V3vXgo!x%K>x9VDL0i+6o9LG#t7Sm81v65N=dK5n!6`ky<@xw6VYL%nT)pR6E zH*7MDuhbtZ0hfJ-rEFN}FVHyXAZA6lrc7F=s|%%ggwLt{Ex{K7MG?R>rIUcrqmz%) ze$*nV!-;fztJWHw*68wCH3&(?;Iq!-olm9>$X`T3^Ufplm)Ep(FRwKhSsI4o?MlCipsD% zr|@!?WSUBM%rJ`ctR4Y7$+z}JLJz?wsSg8SELaS{dqbY=fH;fpyJIL*d-dD9F&JR7 zsTg_AzQhgBCpbkIP1gbm{-L56mC)t1+j)o3O9{B-?k^@C+c5@L<6Sc{MW7b=PB9ta zR1|muOxs%`>dn%#W8P2oYqFYQapXMS&0=6aS&RSk#=3uxC^yL$Bd{_i9~a{`>XqW$ z0^U9a;J;0zJFrX)KYK*{=>6ofeet$FM(u^#Q`~2*5o-2Wka8B7722&oKJa| z%ky^OWBYiR%yHhtq;6s=D-Qq56L^wIjODWq)-JE3!?2wF!*m}uO*kN~Mvzaktfe{J z!fBD5JWd!iY)XEo^4k3~Ub5EspT>IQC=4uc4xEn(Y&?2A8R`Ar05gQ%i7r4<00`NG z6bl$z`G@wXhKIDp=YS?$W0P+(r4Z&XO2_g*0LoCni@$E#zHkb5aLHG;9x`~c?giCCg230?v=zo zpnF&H=u#v97-BFG33N;!3+zni6cTP?i1z6B9o-$D6p-i>rDWfa(>T!hlP(Tx^ojkk z=+pB0nw4Ul_5T9J{d-MOW*T&Ic380~>1028Aa$6}hF46?D|WaI5^GQTmPe?bog71x=-Wn-F@;(aOrIC^%v#-QlB#%2nZ79lX}q zSFC8uR~m%u!ehpDM*$2!g7yVSZX(gam7Nqq4}pe=Ya+lLE;+_k5;)>$VZ1aSX;8e7 z{3(sTa%7%R$zSxa({9?^aU|8VvVes8qz#TN;}Z`(FH#H9rXxsRs=7yPsY-r;6@fnM z9-xu_5N%AN{k0qk9g_7z=`DY?_jm!a!P_M;r-MXjOXHWsaoxMK!x3~VuLX3=-%62P zjG%B%hz^=ZD(--@b@8%?eTQRrQ3_qC&`wcJXQL^&Vo1K?;&Of5r6xut{)aUBL=Oa_ z|66>|EB+$wyf2F*I~J?5G3Q<`h+@wcS7Q*G%m zZKRgjywX@yS0HDH*=968=^rCxjHxpir5d%ZAdrmIZ_zkTgY8|2+JI7v!Qzi=Hsz2T z298z98LvXTRgQSF4IG?iBH*eaWywVY`Uvzoi-UqK91cf;!+(P6G=UCK6J3kS_ElXiLum&ueCDI%QsggeP&1kEC?OP{dxYdQTdd5~ZJHH^|;7)OT{ z-;~kQwgL->3W;#V;r)7J4g#o}#$t#Yc_{MH*Bd@_jjfYb|W1if5GB8hvROHH}DULV(#Iy?#wo{2KpYjzbCkVMPip z!KqGtfm=;2fxk10;A~Geb8Bs8kx%ip*=)Dgq!dbjdaHTswpGBT0lc(GXzBc4iTrSU zj=H1E4?K^vsgX2#{LFAteR^whAJbi#2Jj`4-zEE2Kfa#+z1KY-VIFn)0g432d?LvM z6CBi73stme|CDEqVokPxz5Z!)|GV9%PY?dNx4X7xis1DWZ*eQe^Mkdbds(zjUjm4> zT!$8O6zggNkz(VYKm_4hY=7QZ=Qyncotjs`uog6?Xs4z$>Iru77JU?o;Q-VUC729E zQV7k$=bQ9ipck#n5{MZ+*3#AfqI02j48Qb3Xn>)Fq^;} z^CyHpbUA@F8Mynnc3m?0myirQn$zQfhaQ4<(n0O(cY}U=ERz3JORf0zdoL$cARAl{w=w5AA4L!JSem7j3^9B-tqDaa+W}l3dpE z5kK(&K;o5E^eI64GPblJCE-j)sdf(@qlnKqhV%E|u^53+hV;svy%+mWeq1k}^@rd7 zfGs*mT-^R}_P);p_dtd}YCj%$X8RM_X{U_}c{l~5k=9*#@>2t_%vDG(BWc)mO%1=@ zsUa6LeQZWU;CUw$xtR(AYpNYckZhw;hl(YRT%*V^&dYCzjt$8l_`a;xpb~W1MY!7a zR)3;BHVq;REvjq0MA&^O037+aJ}$_1C&-88zCfyio7KG3?SvH2kkjCOM`v7TXi9*C zPLdjgf(wZos3s4Nv;x}Rzd@O)T6r+xi|Zc8Ajk6*zYZy#E@y=hOqd6;P@GB?&?&w4mGrX4ov zs*CZ7EGh70X*k((-It_@>9v@AHh;+e^iMHEK&qF_h3uVQ;Q2tjlc=yA1u(IecVTJk z;}!4l!it-0AoMkHeUi&e;X$+S;Hlte;2tI%_?$OySnR!g@qc!=5B7`u57fuWJc@`_ zZj(~O8;ldAn68I5Z{XW|-@SPL&-(?offvGlLM-@y-mbfxRvzW<*MBvG1IC-@0I=u5 zOODv1mglxC)xTs6D$~3A{<`%qg9-|imlODD_iQuui&D$l@NtJDNy!d?8Zw2((kl75+t zdvqz5skHeRZuA9q=RuXANt~Svh01x)5{F=HV12Ip7B}c1@<+HN&40MQAiMC+3L97( zZFw#iF;)-;3zkrP=QvkL>l^{B=7Jd1#DG!>(7#K+`#8?aNuwHa^GK(^J{8Te5MP;S z9O;fzw#HhKXE0y%5i~AJbag;T<^(JUp5u3iKX<>E7m7%zP;%{6=PMitm+Tjhf>C?@ zI`Wk`iE*;R?u*+l7-3#vyPA3F2;50IKqT&EhC=(a3BWr~Uam2Ta#R3xoyzi6Hmz>~ zEyoPVKs;8l`T51Gm)pBycRvC}%=MCu*FbhQo+$cjFb&ZmdI%SAnJ3l-4Wl4_gwXr` zWv{!2UoW2R@9pBo8Y@y1S#1e8*(cB=xF-r}b6X>J5PkayQL}&=O%1nb?^l?oSf|+&LXtR8F^u($ov_?1V z+wJeHH*^vC^XU#QI+9<3gMF~w1LDeT2)dOWofuaHtS8U^vHff()`wU4V*lXLt0zx) z@P6Mv*dRG(Bo&c%jX({06r+HRv>YNPq*EA>`SonnQ0e2jVfL^(p=(OS(e*;f7zNjh zJ#iv$a|LY%m2VQ?9AD6{3~cBmPXoX#GCoBSBaaOSnb3Zwr9rvgHP{s8v#76+3Rj$a zgV|qi1kx~CE5;WTrj1Fl@vkaqFXf0Qn#W$bm^$^@AsiDoixW+Glu8>rr)&#D!rhdN zPTN@{d_&tJD|KqFIjk*PlUFKJ*zV(~qtOHW=fuYZb4W(*p(3$aGtq{5U9nCf^lxR` zC#Akq{ws6!XjF^C_vf5w(bQK`lk#h*eNRE zZBbY`!27gmU!8r5B9T})%}H6rCQslc1URW}PJg4o0>&x(WbC!7qc92P`cLLIF=QO0 zD@Zw}(3txuLCfgk5nNHIJhKZWJiuq4$|30xGxIM1jeM!K9zX|rIjZ4J626G1D5 zZrFMXiZiMJWoP&Cljm^wisCa3`N(8JR(rrB@Zc@I$C5N91qoh{a86O+s==qPc36Ez z?If#5ASBO(VyY+(kiL@Sx=3J_qF>aLyLQ#DR-!9%MXD9S4$$_q_osSbJ{YZ+S3|K` z(4Q@Ji@KJCW32ajDg@OKjVEW3sIH*BJYq0hkd;N*;cFaR7F`p$WHzYdPEzwlA~K^FQ;kka-_!ntTR^eJ>H zixL-wG-HiWx3L*{B-j)Od437cM^r@AovJKm99(OYbde{lDOR_>SUDtt=Poq5x{f^RbgO>zh9-8 zrbYWDmM|k>ByndiO|`(r$l1KNmK6v#_5lz~UPIg<>~gXb>s+4A$;qy>4YxacW|c~# z*)HPqpa9wE7!@ss3Sv9~evwSmDaKrjtf>i+LVTh%?oxj1Yf>MvwrXB!CvFOvdBEL| z2$hyn*lhsMEzc;jWlQGCsz5XFK%&rq0{WTl=B2<+-H9%f^^W@M!7>_5?$ zni(OQ&#Vn@&DLExp|XcHAu^n3#tE`9d|8 z_%@Qjk}TY4D_|rWgWhCHPtrk2O#_eEYZsd((P4>RaU#~lCKxGiAwd-Invj98Rz%DR zUqj!a&dM>8rrrj#q&&paf$X2zutdJlz7IX(MX_867;IFgqi5Z7vo)F71E$GTnN8Qv zdE3ul9PB-O2{ze%nuMu1Ee2TD)6kWyB*vHS&qZ{;W_cyDZpE5?kR!|z?)v75i+0|k z@u?XrdZH-u+_IRIq!AM??fKo}^(KPy)A;o74U~J#X8z9L0BES;0lex#+FsAL4f{*mLKKmlic z(TGqHWd2YD{U^hl52lU~7>uvUrd~9rdYoc77FtOkhkz}7F zxdi`FSq(}<^jTa8p_0^Dhiv^5wdl$>8F87FOA0f`_1cUw6_GS}}Jvtgod3&rZxOnYk zF7dXY@J!@%AS~WnJ*(PAP!gs%D7a9WjK^g*&7zV6 zq(vC@SwwGJw_ls^S;R+o1sK-I4I~lS?V!LD0K3+am&VI=ki4@CKyBleDIBd}5CSo7 zGAsepwbIap4{n28A?a|2&-VwT_lUabedit6H}z*{pTpZ-!Ahw%?;ZXB&%8Q6_=AGsZT^Rp#b;{t8f`uDfe?y5XW6eiI5n8rbI_Y7rG*{ zs1D5+FN#m5XVJY@l+hpfz0!i@i#H8CE(mF#39l8q5~BvPU%>Y!vQN}gIH>m zH>zoyAaBTt(sB3QVd9a@bYwnx|R?DfbwSQPPE95{{B!U=9yiN4w8sN>Dn{L}UAyscsBM zgVEWj*ub%i_doqRn$t-ueni0~YDpVVX_};R>()AMWKdp3a0NLQN%M{{8~w8+C~;u^ zfW#ml9MjD0zk+^oXb}%6t7q41YG2b~`fbyQ0zwek-pq8OSZWTc^ExqCYs^+aG0P3} z{@QdXM#XCo*W3u#Q@=)k{SGLoa$O8K1SkHqd0FhqC&gZG+)_Wtw$#AOx$I+VEMZWI zL_m?30!6yLo353rca+oBVt+mrpKS`$rKA*6jYDHbw(;!$}71s52p}w!M z=eTGa|IV-acYaml_1ug$=$Wz_kN42Nn@q1x`Xj`PVsJOUa{;tBEA}@=8(41z^C=25 zem5TC=l0$!Y+S$SU&AoU%V+X)&jTI!sC#bZBMytavxQgJU)*yb_{kS{w){mXbMLOd zy1sW8W%l;(_zV2Riyx0i*LUu15*7X|@EdP`iF($1!1F3}O+r@rIGfMT8U4hwDfszF z?_@5%E*kG9TbrAo`$u10VO%3$83~lo)e+`Lcwz00zE4enqb4%1ZpI|r0F5IJvT+Kr z9lZ6Cf+tel@$8U81!$76?ODT6=bZ`GNJ-2HOW*lscWf2)^*?+FTmQY`@GdJ4>V*%I zgcfM|jnNYKykerQO<9E1^r^=76Py{q61GAVi^g*{a2u%GxC#-#nc1;)17PL|-UmKG zRx&_Jp}Vck+hn<9G03q_)`R>N{dl=ituiX%Ml}TJ!NAHVy85GChdK!IyFD3qk3wq?Lc>4?CFN@X4W2iA)4WVse5GC?6U|};g*Lk#2MpBa47g@c zRbYk6T+zk<;g7du-3TQD!w9hzl0@DfVA+Ko_KLDP{67LqKR5`3$U9m)1E3o&>=Qar zE_8#NQA*yrIA)9j4+!CA0dBkm<}mHYZ-Qa)e4Q#1obbJ+^AE7#t=?c7aH?w`sVw_s zSQC1JC9asyF|Wk1%Uj;(mB2Z_V|+j5JtB+BO*<3Zy{ii|@TwgSW z%bSNl(Iddd+7+@Bd7GkEl$HK>UDlfawauQ)E^<8|f_z>E1cw0jhh(TlwL6*Dkba%p z&Sn3EV^#}%a&n4K)I{~<*Xe0fV#x^vkrE-~WeIltQf@W*GT=2d5uf?jzmZuzyMX zSMh|u?@w^OHtx<8ha7Gu%gwvk>RG=#9!*B4)1q1x4q<-T)o5ZZ#RlJFxoh+u=RE&8 zPno)yba8P-<(#R>u!qD8oIZvBmun=h$TVPbmz~^p_^~~SU4Z4l@p?M(p&cQqk%2Po z;hY?1j_BD+PQV1P=xuPCTY{UO2bG9nM4-zVMcTeYcZ|r_DuJnfoC)BFnjXF2lw&q1 zvl$V}yDreUPROvRuBz#o02~d-gb_3|@E9*|z(*qc9!|INVVG6HzK8Ov4%MsoZ4|FYbs?avmNIrcUJa6|W}x3^2%*{t9k`Gzlr(nW){)27YM1b*e`6`bY8 ztJ!R#Dr$rNw*Xnf|CVweC-4a4JHZ%n;neugHE+h8chTX@f8?||TEYLFX>;2CA7sWo zBIeC0XC7IoEJlgz-0vie${aGy_{{;BH{l;=L-9asJ^{bI862x4E1?n5Kl>bfmalMU z^KY9n@2*iQx2aU*vWv%Kqzb>s#eZ}$?zU%W8~s3SQ(6D^Fc8B5VUv?7OuX~qus6P0 zN<&B0?+z`-q03PwjZvj{v<~~&p5H2tvG)Y0)$`^q9)7xlB0X%}e^9Kx9={&0t^iQm zu&|W;bmM@lcwOs}jD1@|X133)c;UX$ws;Jkw_ZG!^Gyra)PHMlZdz6w0#TCGWt{8A zMs|3a-jt5TO*eeYC@*QmU~fz$mq$sn!g^943mWWM3;}yj3@o*=#J?@`XiuY?&Bo8S zCcj`co*`~%YyYpjMe3YG32|ag-DLQgzZ5rpcXidbozT0(69b>*Y<)Wz9d~Rq0*;B* z$9-(ug#X%f0&b@_7ZEA&jej!|_9d@J;@@#Y6T4os0E#i&T3=P5&w+CL7B}HZ!c@O# zaX$xE(+EmM9L0edu+NY~yZRw}+q530_Xi2WSg}Yj{uq!|BIGc+ta}qivSncFYO)GE zmulpIHuhmZ#0b6XZDYNel=~kc9$Ig6xC8x?AI>o$KSaofVQ_$txV5Cs<^GQZ2Td%C z88@Qsu5Lfq-84FqSR@C7<+S~Hn7EPa$?&EiBeR*LRQ0Mxw}I*MYBf|TX{oS2CFGejtT(+HQyl1-pt7b(wbm6fIY7?8>P{`9=rKnvx}p8cz1 zUIomXv~-at2j{t|aZhP2TW17iKt`k@sFi{+L*H^zLwNwj#zP|dhH`v2RK%l{bi4y#zEWV0z zdFUIXt%P<=Mmc1@e$W+O$UAEJa86%c%0igcb5gU1!a>paMmW}4+t3B(_Wnu=Qp){l zi}s$q5Xms{2Ri6*xQ7#4dG)duVXZO~rLHYH=YCFuevaK%ugjXcAZiASSJ{77h44?Ju`JN17pj!?A`d z`Zv3*0s!~B=a4vel9qsXn{H{**uo!mVj|Ap!Wa=}cU#;*c~`B|sPMILh&|uI^S<4N zq+|j0A70|nXI)5t0-u6ljSBoYnIJ)a@Zlf54=~f*r`a3Rv#2lSUzAaaln9!-+_JG- zoev#$hy4PxcV4VL8lx5Rehz#`9oy|e9~yhVJMLdjS>XTZ&xop(`|jY`)4k4F?`h`) zwrR2Cx8wdvrPzT?19VJS=*jNqn`xmz=fi&gOsF9&$!}nGPkcy5KCn5+^?ZbbxP~X4 z@yXNv@whYY_vmal0t9MH?Yv%M?!4y&QIS#9;jP}p(*s>@5eiC@G4lp9a4E??l_ zm~M(bn3n8IuMS!|olfMZ`TNdF48zyzQYwo>-|0_5LI5_+Utw{e$FkXg}wIfW$&~%hA*YJ58E6MiI&Oy&M5|0OLzak0s)`&97f$; z1bb>*xv4AyY= z>A-eU6^$au^c{|ZCo_F!l=^YPR&P!hr+P>b=CBSHbjpk?h!SR7a;3Fjl}I(b+$m7V1iOB7h*T_5ilCsYYk zQmnw997t*aH6C$9=;UD^9L9If**@Tncrs;@xG9km? zi|+qFPxpIcxCir!(gaL~bj)v)W#Z8XZi)4R}lna@!0-CFs+-+NEM_9$8? z0kDHN(#~gXtD~hHIv|P!34?(SFeYF6dHlD{`toL?Zi^Y0oL|^{%`eMM-wV1o}puzA3_Vjc|Cs2V8xH*lvI1nXcy5 zkPa}X+UNDHAx&v+HSAjSse0NP(jc^&#rVrym(BZRBZ@3Ni0eteBl`}lRg=*@dr&j; zql2ct95jf|Id=Rg(Yo>rrX2tx6O9X;*N05fg`l#n{pRnSo=q{#99NlJ#{GBFZw$lbz0BvgI2#@U8}S5{V^^N38)n3_Z6XFh# zdy$?%UP?NNUFK{tRqC-SD=fbxP`?y_c-s&mR?d9EB;MTr%ig;HS9YA|fkzbe8q2mE zE4CcJEI!x?&p0`a&2t_a)<-g_|vn4(Q@w2tESL(a-s zI;AACEmyP~WmnG171xGY)321A)mB^^WmnmFD~coAO5!A}l*>`d+0uUB|LD_w?zwo7 zR7v(4w^TH#J&G$Qy|T z1z_ZzDrFAuX+fHnf`i&y(TZmZieB&8;$6F;3CV6J@ zU5q#)-j1UI&rLw<<RPJK;mU3afz~l$?;+e2@f-je17imd)W2hEu8p)dXUw25YC_st|8g$uZigF0AE$qR8LY;ppe%)@vj9w#{7B{A~RX!x%VOsL70pzWe^b-`l z-3E`I!I|lLMC;16>LL`fh*g<1kdMEsM;>#L1e%8RTvRYx9fW8}ih()MqJR%0vCq#GaLG29^K59?+W{{{40dHgg)KO!eumj6gr3El zET+p9)^^i+aM%P9#^$PXgi)Z}P<;g>4lRCLtoodK-IPI+@RO9@m?*E%C8p?ucT}h# z52aD+V~tU@&pj$2x0+s1)>%@;n6_!J0dRmfBFZpy#K3l19NNJP19ECCv^Fl4I&>-h zRZ1#%^U2bLQ-I3=gp0?hyX)i)C`P&x&t!rxE|C^IIa13|0fmwdV3|ggamS)=LnEv+ zACjqt_GCb#1yy*691 zD@Rxn_G05c7X+{bHf(!T6`&zj&0d~gTWFE33(g(SMORf!A2C>*D~M39daT03Rf1P@Cukefu!n~NK>c4M^@n|yG5zC@#v=5g4S3`t(fRAY2! z8DhS0)~Kiudkr;XDFIfAd}Y_|1G#MpfL7zt@ybbKD+!U98&#pvbNZP-kzoOtP*SO^ z0UI2zF%svM!u_$G+VovM5TB!$9)f3~Y0!8nsw$eu~qC<|OF4aikowb$_mA$GBgQvVO$3HzH?gq_F zaTnB^fUyhhp%&K`+vD}v(w5MeBFLv2uc_EVWBt-Thlp$bGFQXxrxE6rYYuz!4Oj(f zHBV3|>_8W}==1C>#^ zKd4P^a&+$h#4s6LRZJ&kih_-;*x#uUsO_27^muF!r%2>AsuDpHP83(fsHp}x2*wnZ z#&wRb)f>%f_!M*X61)<@d;?v5UDc7Oz(gAX)Ix=O=l~DNs2*N$o z4^PPUse})*0^?VpbGf2tvR8@9Q`+egcXKv;!@WvW(W$83<2G2RB{af8o^;qjtY>RUXFiJUmEymBQb`F$(b4Utu#@t7V zuO}~3h9p}FVGs+0PML0B<5qoTh+0lG;m|)P?q=R@XCPq(kD^}#jxbwWz+qeV!hAWf zHexz{HFyAl+PU{^5aDr;<;c)H9V|q455g=eB9iu(wZ<3O!Z%%-uCB1D$4b}`zEARo zq6mRTc|xX0xJtY)_3$2};n{idi`m|a)UTu&b75IiOH*x%w*}dyWqPe2Cz-JmpZqp$ zOX;A)-qAE9QWB7_yNbgVyrCG1$pHk1{HiZ+0q9%w67xy5-I_NLhCNyA3|bH!738et z%Be?`Jh(j;gQon0yhyy`7wcllhf(lPBq7ZkE&gvstII(Mq7$zY8ko>ZB9Re9Cxj-$ zC52Q4hzoBRdC&jIG&aj zESCo=3#k>AqRP-v%`1x@(Fes^&H-cMiXTym?w}^skY#8k<_QpEU`K@|9Nj1!OsIQ^ zQmNf_0gF0iB9+p6NBJ}l>4T13Z0SrP;?4Et9GTrx5jP;GOGN2X4@W*OSKwC^hrjxo zPeOqNTU`MIZYO9JcLwtBi4$0AIKbi(P!8y8l=Ca4O${Anbt=Kbv^Qeg&M9DI>Fti? z5Zq{aE|IVtF%CjdqN-H50%%`1qjJ*YxK$iz$XMKQ!kGtKcFbhHGIEPf#iGdN%9way zX_82@^4Pse0mW$5s=AQ@qr?Z&tYR(gRgj2{m6%2l&>ciH#=Y+NRiL8r@Fn9Kmxi(A zDvb_i&KjZ3bg$=6gkt1~0Qn`K>ng$QmdwBeKr!SBYdktU<~d)`aZ*)bPt8J;M$?A7 zH&mY?2I=Qn*j5_c*R(KkE?T z4$02`3|DhyN?kDzrNrkG?5iky9!OZ4wM!4otOJH-6i+GD-7YeV#dUhkB1Nfn3l+T=dhz&5>w5i>8g zviM0_s?&9aYuAY!PGCeCLgi7MQvw9+hHU0$hwNdp!9T(0S$xwtZXcRyNxAS^%PU?gy%_CSN?2{4z zP?~7@$T<`)02Ml6R`=QR9=j zx;=NK#MeyXuv@LMDZS&k;mkfH%u!U|9YyplJ!vA(RWgb!4ABK(GqwzqwD=p3Glt7>Z_hsMgBU5E7L0%~0J>ve z_b=smT%0OHjL7E%S7FCD0rNcSN^5x%>wS3y(-iPo+FCx(#IAzlBrJbG{A4I{&-WPZ zzZQmpT+E1y;$nAzcmJ~AP~?>qh9l;U+KQVa5DMeb=op@{9c(ntM+@s#*!5Ujt%V^d zrAXaIkaLG-(yz5~`HGK;R`4}B9^f409xgq&;IL+E&AM;S+gozNgc^Y0JFiw7 zm4UZx?LbV#*aRgqv53w{2nNUVm#$tPAGs8cGvQV#J)vtgGElAZhRoWkpQI+S%(b!n@fb4v<*O1W?8C)LuOZ$C^oSzX-{;Su^a$7n?6|I^YrMp_YEHR zYt`4aTXBg=k5DCbB>7G|3Vl|1OtsSo3`1_}>`Icjoiab!B`L5>yDow*kw!w*vu=9W8qSL4YpHHN9La%iuk{$!nm0?Ba6Irv%xE7XZn{0ca#dGV8yEu_g z`g18cz6C&wy3+~QuXW0dkvaVN4Kd|b_6Gze7!0N>+=By4s!~;}B1Jl;Cdx)^EluE9 zSwION8qZ&pOscsl4<~Di1EY{2PT#^e@vsuITl`SiI+mCRbq-<|1RTBk*IGFIII(b+ z)B|bZd+y&kImF#ky>N%@{hnvH8|}xiY0AB0)L$y9W(Z9m!Irq4l=6gY2#{(ryIGnRI-%l*?kRPM%a0PeB`)vF?gcidQg*B&;i>} z0-zc;;~Cl&hB07Ypss*SRwNiWj2%Zqp8+X}4i3^QHap3SS)cMeT}sObz>HN$?NN z9lVS!;~G3v#)Uw;oF=Nt^?JJtsE|3R{hgm-!f#`fi*j3|eSy+dV+qO2j60>wK(8-| z21_IQm$#cZ0Jz`7Dv77`kw;}^lrI$r%4fhvrNf&}?W{?XMK)ENbv`-0; z56zcpEeo8Ux#@_NUrl6%uR>1}#z+lZuAd;djTpb~*Btq1S>@n>=y=SXOZi+7hxg6? zr(FyvN0VT!1bhx52m~wVUJ^k;8Ynw^OC^un?L-{N&4Q`Pww!F2_e*I^P9_#O{y^&m z_&#hwey#b z?cj8s0NQ-pBRa04U^&00jGkW(wP%9}tb+k8r=La1&?a~y8ihu?p!>SWNC!Y5owW*N z>eS9%aT-XYU9Yx{S_qj^wblou(lb=wtqjjx$9+Mxy&-clLL3oKafsmx#RI{Lx8B`4 z^Hjh9tu&;LtCkVsC=q8*o;=rYoj5!;`*7a0>uC_kL|WmzJQo)%W(TjG}r5p|qf2e&#Pi^~W= z6ZmXxffwrjadEsWzn}}bI=|xbacry%No-L5jS8vaW+e+npSwSe;IQ7;qhUt`m;KncyIqfH@%SF&Ttc_>hyH(8T4a zuty^fJ$yDzs!g0?xY99zrPIU@D)wI71gbkEw1jl%!NPGaPTK?x&&3nur>b|Wt+wXz z_-=RWo$C)5PX_pbaVlqx^(`gB>g%iqH6vp@E!RiYc%5z@)KgG-*siAcMCHftmYrAY zQ_QJ#s3*VSO2jt(9&Xk1Yy#%5&P{w6DrZ6BNG9M9mU%g~@*>1$^LrPoc?C;?9jrr-s*D9dQoWye|w+V_sOkD zLeej5!6luwwtE$kAg}D=M!n;O{T6MI6lx)7CWA5AJb}y&7xFTMbig*m-h1n1+ z4fzF@j*c!f$t{+^s*3|@3^A#&1QK~ZgWijKUiPrvUOjPMZ+Rfh2-AyDR=7?BsyU)JI_e$}Y+ez} z>#5-O2%aCm-fnMQ7#L{bEceaDR{v&gx!GtnR@+$6*c@2l?O{}U1~yy5ItI9i*u#N} zV6f-dwCn;*9UL8QHjbLq?o}MkwG~sZ)ak1NrmbfYU+Ssc?tP#y(Zi43?j^Ih-HWhC z5Hk79Mkb^!w(~8=E?TWyr=HUFedQaj;CEE-A3AIHFRavXGYAUKU-w7JPTT{6$Je6z z61lZA`6v(oV59W;gmUei>UET$1NZEFuv30=MVOw+b*h(fNh(ODpo~oncT@-&&=T=% z#MP;nxuS#PgrgW48~1Vyd6arlSe4F1uhCfM!VLF#3tFi$Pk26@bDaw7JwPCVuC;@4 zcYHOyT=FK)l+uDU#u_aX!H$b@J{h;b?r@y~)09HuTig^WyIu3CBxx0=qgCOo`HV=4 z@X-wKSfVZx5_lO@^%z~|Aoa?Vr`V$NUryX*QXjWFq~u+`p~IoH4phO^>*Yq1Qzv^< zW^}Po8U}#Y^UEF8_=lYR1QG~CgO**8C*)lHaE0aNbEi)0QQvu#6=sBnNS9I?637xZ zB8k>Wj)Ff0CRs|TI_N<{Z;rkHfj%b#fk4O|^g^E^45pw(8<>byDZN0n`pX1J*pQK&H{MOrkwUj_m}YT1UKI}?f15J;$e^BqcwOG^ic5?T)7<4V{Q zrY%fj3LsG@7~k$EM#4hfSbCvTGM}*fZ_k0obkI%|_gNjL7sy-eMjEw$KZ?~Kx+7&n zww8FO7cc1ELqY;n3h0G9nBqBnaa>LYtYGqxhV!U96Ik<`XJ#Pl9lL!ye2JwV#E135 zeu%i6jXD|6R;wqTLZdr4s*! z3NI|pODIwYK#=&U4sCXxWDG!1juAQs+NaArIhOKr6csZb`53f-+AWWLb+_!QgrXdrsG?1f(-Y;D z#0d}-($W?t)(=lgAX}rk8!=PJuGkz5I@^*PmZg+p9EwW(=fa2@nYB{`sCURV{s2q4 zXLq*BLohD`JvX=77YE|Fc!=UdTrZ!!IFP&}VS*8QS#<#+IPjO(>{*lj@uxnbKbxM3O9a6%T*3<2qGKU`X+a10M>im103X6 zl9%9io!ECoQYz?^d_^t6rH*TXl$pA(*rTO#euG{~>Lag=opq<8DeFXz{4QZ>CsGGl zPJSSj_nEAmIfRu*c^Baa0grWQdc4`}o+vxR~3xyAS zGPq>}ZX}TMo8664npFi?(-9%IlD;^L$yEv<#g!V@LVAQ*;D|b}vKzZO+A=m1u@j!@ zxYe&T0}MsXfL60zB79ECib*p-l2FloCr&^;uWaIy#MRx(F0PfT2nE7;%6$#6Kdh{1 ze0#C=3`VDg^}I?0*ReKMIqN`aC~&tbOVEq&ifc7awFifTX}MBs_eUfH6j&}Vt+YUc zH*iFEIO3wR#oI7ihyVjTUTNWC&ZvLNF0G!tK$EH~#6alqRUnc|SJ1{o|4fjNV&ggg#uBZ0iEdM+$U^>y-=hRG1%Ct}t^Yh9o5= zb_q-KCMyfWzcD?NK}tI+z;qfE#*S;uP858==u>{`BtQ|l_jy!#T3e5`0y=S^xV)@- zoZUhftU%m_6R1yRViEJ{l3%Nvs8uaNRU?}a84kPi51^16kyMmTicV;zYUX;(qbX=% z+nySSxS1fyFVaXF(|g>Ss3Zt4f{o1T3Y+8#FL{QoaGHZP*~hO?g9-;oBU_vT41JUp z#~pVo2K}{-#?sOH7HPeFIJn;PQzQj?L`g34u?_6S*H=>EDcg!J}P-+ZrcQVIo{=Zalm7d&SmX5 zao7Tzrc^mw#vlQtk3&8JGM&sne4!2FR^PH%AKfm9D=Tq4O*pC;Gpc7~7toR0ReAn8 zHcp50;rTVZ*he4;92j}@GruC(sHseZjL~7s#gEvCZe*=}OIsx8H$Nr9X910Bt2Tib za9&UF&4F*Im4GwygQd39P@7stE?v6ad$e^MdxXkrYKYF#5}8g-F;2DZ;^&@`T8me{ zW1ZNj!s~Jabk7)6FDwG<64ybmfrcMjeU3=IXU@|5pt09_XHLEd7%1@@FtO5uJFX?R z9E)t1L4tRLmFQHYT0(R>=Xh^_f470e5!Jm;12C5P{$-HgBdNUW6iz0v)WqqMWC|wS z&e-1L%5u^PA!aWlAK3dQTCN>=SZxW#?2HA^q@3A-FtUOikmPs|mG7S8ww+N)xilDZ zB&?}&JFimmIHLD>sw4YNy*>5=1x{j+YFBpK6w2L6iAz5e56Mr3^q%IdyK@Q#7P=IJ z*PhXc)3CVsDL-6FK`wl-U#(jvJU>PH=%3x3%f>tZ5J9@yPI!vP0X~+sM4!Y3$_Zfw zcyX)p>axRHUkt<;akq7(_eU8iqDXHf%a4~$l-KT8EqAtIsJ)T5fiGR!AXhFQu>LYm zGCVOmx4k90zglOa2y%k`=zEbsiVCx^MJtyIIj>%1(V_u#V1&zH5IUD-TC(sk&*k1- z(O}})U$~9eLD6dgLk(G~UvL8k z1|=NpVDvFj?V~LQCO9Y8GQZ9vuEnVe2^`||jSOwnx)048_7{&i_OSP1kTHL6-@Z6t zFPOhE(rBw9V_&eHxd1Hm+pow>2!xsw=?QZ=T)sl_GKw#+%~+L9YQa*uBiCB07v`39 zu|L1dA!i&aW==s57eW=i7Akjb(H9A^g__0#GZqETsRh8Y9u>#8E~N!CLa{g_x#6CT zAW@oaU$FuXxUd`mD`W{uO}+6|KzfG6u_G1Z#wGUwNjuBp(c(6iI=E-NPwN5H2=usV zU9LOgpz!1@_;~r{o`f1PbN6$tJP2`iEn*_p+RcuZa$^yj#&yO@OI1HATo1Iubp)BvQj$&)D;t?9xM(+3QLww^ z!c8ms&3Ga>x5zQLC5^WP`WRufEWD13$-V-#6fO{o*9f8Zv92}xpLRhXsmJ>pxgxwhDYDX0p4EyON#Xb$&qO z%`(D)H12{6nPZ)_GuKdSx4w)uRd1bN1ns>jnWoKlyH?tV1Ky=dnrsY@Mu>43L%dhT zX!Z%5rlb*fmthK6XfFtlt<}rg1N+pSYI^l5tJdY#lDk)v<=C?CWe_gEi*@gMOl0Za zknY`v<622q!!5^Qu-SmU?n}Om3x%t&U)I;=#wSN0%2u(A1{)nI8 zT!LH*07(XT@`C35Z*1*Sqvr2gXL+wh%3Po|QG0Bpp0cOJLef-s=^O-!CyddcM62UX zbw||q#)ycCp^S9y7o>T$?BN!bAnR(TV`&kfZ*USGs;|J(JX2f1B7$flgXNXGE!8CH2ULxEqpbqQi$zNN{k^c__eyBYeW{06h(hH z_dFx^8lM|#T@Js7xoyF2;N63B+-aRUxv;!pHEV^&MTa-60A>eXibx}1B8q`BB?2Ft zk;@v>ngXL-fe_pgp9mhFw)eQ*rv_BUN(`(+xfDzQ#8oFJIvS#}9QbX?8i=4*6486S zQCuTIuk=cgM2096H#SQVPIM)2<^J74u~6vzbF z-OVMeyd;n$`K^eJ3S(xu%bDxIj@emWhnKSSXh*er74uy7mG$7A zGh7SqwF{kb-7okA}YCmV>x}1s-gO&Z!?Q0(D;;Dka26O0e^^ zjbq6$+b4{Jc>h={x*$FX8QjZrXd2`6j`ws~q&eUX<>axtWEsmNX-v_RAzJKFTXeq0 z^(yZxXV=3ThpmuueklGJOPD}JM3%70TQ9U1w9VguvhH{hUYGTq?WE9QE6DWB8H^mfvt!M3v^C(P#M{pr6TX}^yJ9UJhvt0l3&@EoL!R0 zaRWzi77bj#^MkSd7{So;cxH)OY~z!oyzk;(<~lZdA7FW)2~}KN&fWTi;ev{!24e$A zu16W-j@af8Gy-GA^@hEYBwANV2~fa%q`301pDo>52@5)ot4{6c0B09}C2Fs|k+Ypr zu5{4}2s*ha6mg}n995<(jn>=ET@mlpkaBqwd534`K^&Xy?Je4`3JHA73q;0EHLIk~ zU@#7gaEc6u8PkUgFj&z;7x6=q3WnGa!6vo60Z%Msl64?izT(1G=?6?9E5z&^C4|#V z+d1k5S-`%Kkf3%ET2q^t=@|RdB-<(L>>Izvc_# zRR^}l^>yHB{4v4M_yd;pQ{#YCi8h5RM%m+nnaH`a4aL`*zX`xZR!Cpu9AC`S2s~vk zpTpv^7siJsN7so#mbKRM)|PzXKRO!PLetOc@YQPfMa}IGO~s~h z;?`_R?~s4(IqjOz)Q0Lo=4P2+E-X~X9(1<6@U`z0zsjeZ~i zL$UGYHQ2?KK0Ty$fRi=4b25eWK*swuH3@RChB&FY>W7DzrzMj{PIr>i511!Qi4Ksu z-2)dj+dF2}b!@<4enEWDX5DVXF2^JzG@u)y=r$lq-BpHk8UYDcE`#@zI@~|dSpmm? zFQ@?_U{H~x8P!-*gj`&ypA5?qpHmW{2iueOMliG{Mi(A{J|j=WRyE1ty;3o_C@Xh_ z$@RYFb*1km>s?cgnPFRW)_Oxns427YPqB6BPAaQL3$?)vv^`>7VL120Cxv9#HMYX~ zI;$<>k4rXToWk}a#YX!3rDg8nQ5p{nAt|oGCBWd4VOl7~)~?B8`EMq-J*(Ffy@o4$ z@bXIUAOn!T7+f?y?WlaD&jTVAER#o>cK*itHFb>|Fd% zv)0&@8h2WlOpX33E?R~aLadD($Ah28WVpiIauBDLYP=NDy&dN>^->M@bol8TWu#u1 zow;urv`pOm?G%?wJ=g&C+K+T9P<5Z>wukJ_@($$&^>qeN)F{WERVqEsgA90f$Q0O$ zoDgHgOtV^4*_Jf5T*T*2JTW}4J8ow%$vV8pA#E;V7e+tY#lt1!?1!%zzx%+Iqun$Z zzI>;qDkC-(6!uPzIyOY_4Tre@NfAQRd|yxS#F(=)xnyglr7ya^L+YU6N=nvovxG?( zyl6V+!w!j~!)z3+^NmtRd6(5wb;hy?-!e>D=H^t6ZgNh5$wx^?4~{c7vr@-@MZ9uo z83OJFEHs5{586&AIz{QIOE;84ti0MQdAdnMC@^B=EI#7mc#WABWUxb=sesICa3ig? zJ_iF;2~#VuW2-wzRe&AMm=3(;tT6IKq))e(I-EnD4sb&VQRf0P?*wC%<;`Zj#Kbx@n!=915qdhI ziq?1?iy5`{@!s3@2l~3za10mTVF*O@X0OzQ9Lngt<*uVQa?xGfCG)vSYV1f9nJE&G zm-FuQ{Dr+3_#Uz+72j^!NUbqvs=R49=2z#owiMq{0?!RRK3gjIdtrR87-pujB(bN(EQV$s_WgW_3B+c zQUTHD|{bM5VwTB8EpiRo6b0@|#J zw~VAjI$sFSkDV&qy~eC)f8^qUf*P&R4ln2~M2N=S7Czz(j@rrsm+Una01}q!u}GS) z)>l{THMo56TjU0TRSqr8-Gr5^^$auJ@IwztaDPuoCTAR2gPuFGyzDarlo*Svl5qIKahxrFL_7<`?uw5tuj+MBK72j}^cJn2qeuI3RS_ED!|E3oc{)+# zS+c_~Dp)u+NRXG^{pKc5;ianB#M+lzEI4X_ozdng#cNbH-FZwguWYpMUVsRRMp(Ny z!>mCl#?z!`qP?17dA%Lw1z0ouV`pc)6r~YB!Z0K@-95{KrBD+KXdp*t88>Lqt;7f4 z>+xYtzK0AxrX|x`GT8T!UX2Yv!cKA^>GeA>K9Gawj(DTYwuh`eGLbcHIQxk4XZz1# z?#T@B)KZt21A*Z9PsvtR?4E$=49ZC97Guhj?n}2Sl_LisU1!kXgODz!dzGZ?47%`V zr0ba~geF|J1%v#bkuCy36i-snKOTMu1 zm2weD(srRUH>I5Yhzs4i;OnYsQ)*J_gB}bWoF{%RZX82POXMPW1UGP9oJ%r?5{t0) zw7LSHHt@6jq&JaQ_7;mM(P)b964BZ+BqEi3G-+@T7SJAm)P+K36l**_x) zIfeMsLJ-1qhB^nr3n^h|nb)MqiKPRW5WGCF6-Tb%6tUvWKHDGiEO?1wmUo+vKEXXh z$;|#LQWS699^@zV`Tw^B$nE_?iZMAHX!35F6P8$Es>#3cJvZ=6N?JK(Z!nhDs%x}# z)*-Y=Md)dRe3A#7oxWN+V&IkrL9RJ3A$@E4%QL zFRyRaFDNSiyjfD(W?7`JGqg0hTji|%b+5b-ztbhYwp=eSXlc{>Tqx-7;+@Wl!wI=L z>}5fL1C^h{#z9g~rh-pr7*u7bG}h^wlCoei-YG}pPg=xnm!dArxLg9N!y0l~?J|~8 zgzs7j5|GQ{KViJOozm`#THbCpLA?5o^%rEmba$CBRC899=FnK|qDUolg~nDn#5hJx zMg*)RruiSRQysRX2g^WB#n32CAqmcegzK>-6a3V&_Da^`P={n38Cc02@dca1$GA0d zRRtWr+g#j&l%L0t%q-7sg0}ZU$-P}T*z!~MUSMZ>x`2E zHnusV%i78>l7#cd{|X@CVTA@?nO>y@0k%;(omHVi?=_>PHaORX`iiw#k8s#DF3;6Q zd5xJaipiGPvGbSj6&5YhS>Cm(G-%U4vRke5jLtg|}g)z6HSuR6EG z*F~Dg`x!~$(NJDcxZC9B64bG$ct4H|nXc1y&JA#QUaq&t^cW6X*#s2Jjm>@_8airO zPcUyk6*#m_{XWECPBPymG4to=eFv*LdfPn@NIB0A?{CR*zNjkL-w%{+MUN-%1fNhs zR}H!mLzVgu4y4=cxt#OY550Q*1l%!qs>|Er&~Y2Pz>9ElEZ!3^yqytxDz0E9GMttCLbnUJSZf7( z)+dQV7TpV%urEy@hp|NTPL-}gxlpufEBUGK<`{rwU*3G$bM)ldn`I9mjPL9HO?V?W zHv3KhgCs!h+^|Hf74)W1VPa2r8>-?egV@pWyz&%F+up~#PrM4%N+fQh^WF=} ztNlQLfI)(uoa`d9>a51^`pO2t33_?#h|dm`f^_;?qce(^sZxbv6%zvz=pAV&kX&+W zvHNAxRLH}QN}(LJ=kr(6SNf1zpBT6Sp(x{Q;XD{@7HqZq_z(Tv)^tM!UlWJ4v&V6d z0V?Xl`o5$x{|CCaM!uKK7x_hDId*XiXO>DToYsoO7uH(aOMN{gFx+c`P+U&V!7qh= zD92HLoJ)}yi_~bqG3;3F))RHx$8_a-_P{{pz6Tz>K0kK;!3XX`9>zVVA-qbA9Qq;> zymnYF(%{=NEHAX(0HPu8`TVnPAJ221Jy-k|Wu=c?iCj5z<>nQhS~Ui$*Vk@Ziyy39 zsbW({CI0H}7?lSHP?X>#_BYVr$c)FK7d7fO$WFUViGp-37$w*WgmNzyI9Fg42Og~8 zUsQ&F9~@v`amxgPxbDZqM+I)Dh!0sF9C)k(Xuzr&mwsI9;uOIt4voo~wi4yY z2lV*>iBULl#cfI9z`^VaS;!o1Y$UEW>8mr-Z0zg`h^cL~<-B!Hj8;GR;l>ISg#^WW z0hcfcodu<>F!UGj1Cq6^MV_@Cexu(MlR5y-%CI1E!~ ze#1i~T6Pf3z-Iy;66yGP1snu-HWK2d)haDRYf+F?J`WiUMV!vd$9hVmUDdmD#z*GF zUrs6^VtC^TW>@8MYSf>1MN8oUbI8kye}LRVu`CBb=B=$%a#oaq4Sy1cQyf-_mid43JrcoJ5B z;ICu5Ouq1<>hnCUW|t~gu8&X5FU-tNk6a&~pIaCipH-^sGDSo^BYLY?kZT%As?M3d zrz~T;nT;gSpwdsF5!e0!SP_j=aH&j&O%<#K?f1&UHj6w*bc1D5xRQwQ)hBtxcu;Q6 zr3yp9)spwr8+Yr`81%Siw6$~s6Q1;_=fEQ{H+_9}cywWQdU`$vy}|%Q%;}uKM66+D zF#-oUA=+TmnFw;D_&q8Jx9DZuYA(|_rhGKkZE^w~gPpN63ipt}WLoTUg(|YVj`<6B zp~Ys6n;8H{*JQ8~CE^0JLl%4Q(Fcz9_4Pk=?C7xp$Yf|k$fNeRye2Dwf}Gc%*Hw9_ za!R>6iYTqC7vhXw_J_SiI zFD0)tm(@i69}-9SN#5}aLGIMqN6wx<^YGb6@>B$?>UN!4Ms6j+-XSU+uxb_D>h4kl9p}B-*S|W@G2$Kr9zaZ zHhSrBV*4E!z|okRIwZ8dyaF3mvPNxP%o0q@Fg_?qP-G#37|N-TjBm8iI`ehpF_Dlo zS88E8rM%h?DJ9qIwsNYh-xGPa=c0`KWvGBI_=$EWbisFAAzw7VWOUPMzy!wXJ4v{D zc+Jc1`qFt7CrHcPOVLjWb!7Llk%|Og#+Y0dng>QAWQEjhwpE)P(9(`oA&~9ez z((Igj#f^z%ka8#%4@)is6BnV3eaMAV{qss&?<##>8wdxWON|dw`95)6%Le_(s|&t zm?2mI;r+(OZRMWh2l}o}PmUfNSgO^b4d_{8LU@zB(Hd%5W|ZlRr!CbfFpgWnIk^xPCb(lh<1on@o(piXnki}iYS1%mTv z9Vs9va~eigPM-0ptlPV|crO4)&Y$+gA}QMj^IRz6m&_IQFfdq2sQ=%P7 z=Q<^HaRBuN-0QX+m1D7*ISrW+#7k0%Bwu2=R^sLZPn76b=Sw@E4g?zQXu!NdbXGW^ z=|RL2YS-O1Q<5&N$)sq7Ylo~*J{!Vthti#-gOZ7|!FvE>X%8b54Z6v9dDHWe7is4; zS#I{R0|ZQxno=iR@sA08%Fhi5vMOi*7Cl1>^m<Wz|lDHjmOl{6Grr7C~l>(gqUagWUhm5AFr4~ zXA)oYeaoBJG9E9Xm_p`$EvA4liJUwns|WD&U_Oi|rp`f>J*^I+p;d=pUJp5^f4E$^ z--=clP^+{PVO^(yTL$vns`NbY-0<|&*!b1!vqSUa(^Jp)ppn?=0vFtXNvd!P$3h*) z_zk;bo_b7*2O(hFc}hyC7AplfFvZ&g_nt0IuoO7)M44WfWWB338b-XoN|p= z+C~ZkDh3Z?2*oD%!ze|H6sZcu*fA@Hoq;-tkb{0d2LkT6;fu9_8kE-dTIhjJFK4|Z zJoT;ZHsvT||Bq42x5?QoL zvCX^$bo7C2AXy`Vr9Lggbv+UN^sp+%J3KOwl35a&@jg4QT3%M#jX^W9X5fS1s-db zi2`Npm@aC+$Wx`ozb5`_E$HB`#z0HfMLOF2TBcG_9NbF^1c3^Ijz8sV2|U3i5*yVzW6LTXwk6>R99!lR#aDg^!7ZwFN~BZ5oGV%S zwIy?JUj0Y*vZr&BcL?BdH`8(0bA0DT!p*H?GREY(`>N&%n=KZTwxEa!ht|Z(&3ML* zO+Aw33>DaOgPuY*@aJMPb9}I&arPu->PTu4TCk|9!dk@;)Wrv|sB~%GSm)YPl`1we zg=I9F!~>s2Mz3xT3+YZw^b1-bM%EP?InL0%rXiPE=L5+r$belJ5|GvW1GS5sh*n%G z8kIFGhrLi(KjK#gRdPd78Jf5;^yJ*a-00Bk@U?~ObEC5hGeh&&=8$M=ngy67$cChM zAUXR)A+WApzq&9!H8wpvDPr$}{z4%=D|Z*0t0(T(+UqA)TDzNKWFc5lF3S{Rgw5me zxdvQtHwAZyALS*eWNejd*A6?sd>5=>bOp;f z@(@L4CdTLH7skdXMv=p*L7btVom9#oh@xh8!h#YxEIUuBNV^NHYm6{73$Eh&vU{`f9B?gCMIOcAff65Pz|{*APtuN zl*DnUWSYes78d~77yumuYxjiuEO$p5j*D} zUclAk$Mcg%^l*{~`d9`i;{4Fm$k6P_!kNghN@=RZO zcgW)uzbh9Ro}PJfVdmP<+$exQJ3a$oD7_&S$Z5{-!o<+j)$2o7nM}tBDc9=Tk3MSr zigQV8!9DQd=HQqPk+_Xc3D+2&nOPXUIXpT;b4||nA{OOE&Cbt{?@@?+j++cRxByfn zp(m(?_n++%ccQ$7daautbS;jWaof{7?7P$rp0@xnEMOew$EU87s6073M^KdXVP9}y z!~mKDz{Bq$q!(1)9gm&E4TZ(_P6Nb~G^3qx0?ug@>cj?Q1l%lwlw5i%B1UY~-jIkND?(Cj#;VP0w(9Bh{WC>~*<&~%Yi zq*Z{REFYM)<(U}2GCMT;WHFOA%BU3D5RXrO2dcw&dTQcHnbF2*kU=;;WkJZNa=@kv z_Wd};lV`3(}+tAQIe-Tr3Wr<`A>Kr6&{w znhm|=nRjwd&tHSOGCVdhbaf6GASt6p!m%F+5E&bMjOfnI%OE*0Fk0HpkH@^_qygHX zcwYgEGT-#F?)0445FdD^61vM?d4tM(!zOrVIsjn3wk}SLcpnMaz#&vdo!!RC<1j}@ zRMPq4!tlb(#B~q@{^L`?rX^(5c5OLg6MG2;#`^(WpCKW&b~kC)=nnQ@-dJqb#kC=7 zmr)ZNGM0AY1|;R66Jk0-zWs`bx}BD_GL1Pk1RUBHYZ)*?5!OMqQKFzOhKgBbm3Orw z5SIj>F!S_IwpUerhp^8w0dij+ENfE=5GKY%leTQv#b168s|Xzz1?);gu(KQ^o*TWw zg%REcT~WF@4E?U){g^l-9ePyI>R{QtLR0f0{U=FMU{6qtr*aycC@m_eUtB$sEy5+8 z;7n8e7BZG}k~|=C!`l;oPzE@3!MlVsK@m%vM5RmLLF*Lz*ht|-I)2)eXxYn7P)yo2 z-oo+WHQf&7p8V)zS<)$85p(iTR(U=Vc%pznBi2`Sm|jOuEg!ekS?jAkK#^Kp1|*Cm zcf0A#1gyVSMhK&m(}@I0#6LNip-fv&CdS}uRc-3=12{&&DShwlb`xKwm&Cyddn*p( z8)y@&_j#+*?pYDw0tKAu%e#p)w3bJo_*yjc$|S|QKflH2v$Vx7XTqM1r3xmxxR9sM z(Vd7G->Au{7*TK@P!Ue^(8=mYa)?M`1kX6TNm$y3Ewdy$^xfCv`0&{+VJpdrcgO~Lu=DxI@kfLU2Q(J?T3=1@_bdIqXZhxQH>JPM83+QC@FMm zaC#8Xygeg5=V6bvFE^&b*jlZ(YK^-33rd}O(#;0P)2z$Nvt+d@x5F-(Q7+Djizq8S zqxCg-;p6x#p1cRoYW@P@?RP&EI;asS(b&oQhp&;zIBS(_%`Hi(n=oxdaX5+LmX{<3 z(r~klXA)V|`g8*l{YUNKmZUfk9?NTMgt%Hl_MF(^R@Morng|6|g}#>o zDukA8P_s&{IK3yCEzt3k>gi&4N1ZnFQN^MgMqE@BIYJWEgto?q8mgBJ5u56VO~;aR zYHjXN){bEB8Ojv_3}Su-d^9pLZS!vP73aa!pOG~Z#s64gP{iU|1{I2=SNU)<(NikA z+o;E5mSl=XO^UYDa5xHp5*rc}56%xrxvEr;1$37p7 zB!eLm5BilvJ-{N=Yow8qO+J7~Lki!5JBAGm))C9cMHd)#lY$be9!et6C^*SfH%C?Z zVMja51_>Pr46XcNJ7lC#pzGZ`%jK~4Hb;P@p27&ZH<(JEvqf$;KpkK9vGJTeo~nRhCa--@QKO zLmE`4B#YCqjK3fakGU>6XG(DwnPP1B{sep>;~`#aZ*)m0#ri@DpSHR^TAPJ`WnS2P zKz{`VJ7+`LWbo8J70OtS;gT=6n6m|A;jvHvJ!2zsLE>5B`yy0a3Xrd*r2t#xBL=}{ zyKR=on=VI60mNG*YZ2Y!%~C zi1B2yQgNz)Ck-39x+KMSBn`U^1$vGIfh8r->5wD!tH1!$gm|w>G~UkY;@MXk0UwPk z$&pwg8UlIQc~0T`7+81k9~YfG|5UOYKTRPUIMS$it5Eu7M(0-=8LGxt+lUbAFY!oD zGne}06_ONv>eA1#<4*czzRWjAn^tBGi87ac4h9b`!IT}2AE#*MQips!3;YZB?1Y#! zNkJ!xPZH;ng`MJd@o}80TA>&Wnf6l8Dc<@Gz9}1vF7@Daw-e`k9_IyvRLd^j@B_>> z0!oUh@7U4 z4l2>{H=(6@8Cb~g?-gkZ6EGCq+gp(prZ_Hq?(fGjEBLMoH^9MLA30YPJ}^*tT?CNu z?+aL=#VvqQ5h(e6Dizb0TXi9R4ltH`9$D)CPzp!B!(NtOn57s6^q>UqK+(%7I>yIv zgAx|n6${9oQSSF-1G)CkdMHM9l#`AZA=Ht?P(jc;bL5p{;fdZ{EO0Yn@-e!y0+Jkt zbP@Xnn&_^|69+j1HO+^cHJh7Q>hY6J*`O{}vsZm-?$=VQDlvATDyxNTriXjNQzgEu z57;AsfgeG9s|2EFrHl#;;T3qWz#Fux%fXHbD{{sl&#vC%788!P)f;I<8Y9EqCxZ+r z$l{9Eor%y<{!Q_WowJ86G4#PWJD_!5zuJa)c_r;|Oa(S8DP=Rz6IRHRtEp08@crl@ z(*{d#Cn)5Q4(af_`GXn@Q-!&be*lPy)2LZAI=}RqoENuXn)w7Plzvl=H^{%BY{~q-3es zuv)8bthDk2u=~N98+WTXUW9PqI%Y&}1D_z>RVLfXP>?4tjviCSNACB^!F90gp@zW3 z{v8rTY(+T4VY!e2MRsKBG2gTe$UFyQfXJadt%)JPc{O^3ph4z#-8?P5F<2FDw`Fmk z=L*1d+1!91hDasb8EYpc~(TjrS? zI`2drTyy9GO_xH%m*h@9&*^D4IrN@UIQdOZ!r9M>7zWGFKkO@}$=Rz(br`fhb4WZE zpT6IMk6j-oAH7I;H@(e0L61unakqnKD>? zo|VYx5YbT<|GU0G7Q5u>(qX&uZmPipk4QibP!((q=+9x+k9`7|D>xPZKCnnT)6&>Q zPN8hj!c9E_X*O6t7|g87QC_xp0aF;qdt&RC6{dIYxeqEidM_%pyYm{)V2P?3uV$(< zlHLf!%Tax#bV9J|#xq(iGxBmjt5&cTQ7l^sZfDs-Z{-q7j04f&)fHFZ1Kz@0JwE4w zOouLB$Gk^OYanW=cSx(B!xco~C5);XEp?%`h3<6et;rNjao7$^i&p!3;MzqWsxY2M zTfJ<~ao6EYzs#HGNCSq|XTPH~m|lH7BaJ%9aX{>oGLJ-Zcwl0C@j?avRL=FE>OWf< zZfx!1vibFPrEmCH<DrCMsJ8_|M_wHR% zOqPr|HVD|fjE)ch)%u-Uvr*>}lM*kSn`s{BKlZn@xPtR;*S49ea{tklO5f4eF?_4z z*{q^{D_Q0R!hxn2_f1xqr&ZkmLTcKMGTM!bq_ywxyWw50%k3&WY}PO(j<&M7jVwFb zx^Q&mLM5x!(F-rpAXa zRJN(Yx1}pOpUI*Kp9=a+EEt*@m&UTrHg;5lJg+hTU}G%1hGz!PO*~KIS;y1H<9~0! zk9+XE5zm|Oycy41@bK>ir2C_H4`v@k_6lz$xI|AF!^B79=%-y-xsc<^rV zHw=4_@7vjOV`Bruf)Jjah;Oy4@*VtpYY|tbGA({HdUG4N$E!Gy&GW4-HkWV+Zk8$k z%`zd@$g{T0Jy{5Gl@KNrLre=?SqTR(mI-g+6>@R^?V*o+9i9`6+ai)MB)>-3T)Ybj zaFJ2`jbbH46WLDb3fn7nB24ORH|pD)OI4z5##4G20x>aRvTVmH!f6Slf>~?3B`3@g ziR-*y2sN%4PA>zw+sF+VV*V3mkUKFmxaFT^?oy;j*z1VgTiM*kxr!zdEZSkzM15#K z=ocp2+mN0*9e34L8mkp-t0q{pg@wEe_1XpnB7tEvoqdij<67IbYGo0sg-^9S??&}b zbwiu6#8bNKC+gKTk-W1&MpSdG=hNVvQuXf1$ zpz{ddhIrEHUqreu;{8|fK7o4H-#2L0X4yJ^{UN^n7d-cFD{p!`LJ*|&M&lWVLI4qp zio=Zf-NZpQ0M#ykZ*Q4ofoUUo5Qen)9lje0#C!mN>!6)LE0)Iv5hE(u>PCHT8*!I` z@G;zVdge%x$_$`gvYLqy5>AgEX&1k1o6oUvj zdk_~GZ8dlwfz*BJ*6>q|AI8Hn_#5+0#)ZEx^~nfv_$0j#?%GCUiTp%NKS<{T&4sDW zElh<5CqMpIVQ`T@1yxW=nkN6m`T!>$YyyKyHN)khu#jv*M#XM%uz}h#Bv7P@@^cQ- zq|&}C^+^pfjwNZ(0&Y7;OHhW%pYof+A-oN@*VZeWSXu&Rn+!l8iZFn~eQ_n5@YF;$ z1>8V2VrdmwotGI6B%cbL7~>wuVcZX4x6o3WjTSC=hme5>^7p{-2|~WIMCBJl#?rIw zd;ijK)}DtvxPy87nZfKs2><@Eq3lJ3Z~NPW+13vaXaD%04Q6eGKl2Sk*+&pR`fGz3 z>CE>v2c-=DL#cs+j|((4$U59k6%ycw%cU>wj%6Rg^HDtfyI1~&Y>V(?i2FDm(h&ZA z62G7P80Zwj5B?nFON2l0GlSX72>&;PABMd6pYT2Ve#o78|1{!n{5Uk>F#9dM z|2@1@p8V&?|2e#W0P+48m;XQGJInn9%3=O>r29PHe;MHy5gxbvKZo~M@c!edcclrs z$olP%?PFU&uQJet3?@*>4y64ewq?s34G|(&H?~^@j#2`^&yg4rSQHLz?J24Uzfz8~FK%{qC0E^l#ylA3zn z#SKb4yP=Gu{in`?jiN0grJz1hyBGottRLc$NV8e?;&Z?;JRigJaXg>ILwwu8c>NN7 zzl7)0cs|45cgC_85dT~F{aHMp!$Usyd4yYt|02Ry@VxMW;p|KNh9`SJa1_s5@w^?+ zyYN)-9LK{y054lY^Er4c7=P}vWxVdTMIRf@z?T?bTO}MQXSOqZ0&m-bPz|28uDT@m znNi5cX?-%7r0AC}fxqFoi020wsyHvS`688B{JN+LgvM1HtNZ~$0~ge8RG=P+x}HlY zz=8F}X5Uc=kN7(ZOv6yttBXxnf$<$|f#;Kp*dmguqcGC@>$prWF4KD%D=jER z@WV0bi;hM}@4P;007`_d1XIn>COiKFW7)rX3Hm6)2me>lIfVBhoIv>f2{E>e!m~9|jLb!$S>6bw-5uQi5gYcgr-3tic@XMgH2tR@N4}=1@s5ucOv`*!t;p#6vAIX_)7?X80lX^`1QX!nEg7!hkgxo4&epV|2c$z z4&iSj{L4uH1%&@M-v0pM-$m$uuv}-0%MImkxG#k9&_?U0E#&)hRZtFmR7e#==zMD# zq$kK%4Wenb3KI#_t~MD9+=BWbVPoyoNxnDMfEPR2CaOSEU^!{sUFfPSczFLT^;L$n zgkuiZ>cTVFq}H?4tSQ%N972pC#tN*N*g)ZcZ0s(qQ)`BEM3|sXu&x?p(`^}k)}KB4 z*Jpor>|gxk<182Q`s439F*5(h_q}b9Z!drOUkobKray@JJSo$@P?2LZ1?iV+BtlI8 zgJao`{W^FK!rtE;lsNwLTX;YEX^aKJ5B$SC{?Ft0zeoCWzcHBoFdqIO}aR`LeVgzNei1!VHM*X7STlQ zT!`d%Oh$n%4&xQZ-A|jGD3nl#7sK>~>`2X*L|+9&6=B$F&Q5Ku%4Ms~#%7-K!jZR0 zr36Ba94KUDn=M>RW8|Q|vF$ zbxRe~>YZ9+8^}W|I%rWxM|4c!(U;fFREQ|6JMIEyKUrFUlN#&n)sMX}FSNxz0OBW5 zS`-tXi{9D7BzPK+q(L5>W;6UA9IwIrkGJae2HE9Pq3b-8k`YgMpT&dE7_a0#3 z+~_=yU&@9jrsp`o46o0QW;lmoX!fcC^XX^jpPqSkGhtMUV7;Z zUzneN$I{XV-kE*xOqPB41CVd@w^|;2k`tTo=^P6vFuZL zUc&S1c=$It{e<8zY0!J{&hX0g^aQ{0_tf}Ab`{4|%seq9ark|8V)}}_;I|wo#r<22)bxBdF*=pu zz?R{m`7C?)DN&pmkFi058BZ^Co(hXHFm7x9eXnZa`tkN-W5-?e`_n5`px5#biX-}u~M z_I`v*pB>CTfbhN0$9@#yFus2n;g2BxBMAQRR(xlffAjx>y$9hRA>RMUWN722 zPP}YJ-%YGa!T2p2Atbjd&aB9|9?d+E-)RocR+xcdZFSYZRd=e(+lYoZDZB(6il%Mw zE^aPkNFfyb$?P~R0Q!wY#(WJ|A)|C~o$Y!JVw>;*%}HH=WL*^h&6-|p22R1&RGj>0 z52AK#SHL<56fU-LaTmP4F3RUFgjdGhWhDM~EyNiiEon3(2Y+Ia@-UQrv-p2VnM#I&@4>TCq9&=Ov|{@}jq~=%LGf zC~$MBLH=YqD)m=w2sA(JC?WRTS%f8OEes0Ur5zMb&-NjK+%Kwq1TwAZq9!C95f1=( zSJ|>YXfhR1Gmq(SQ4{PpSAWoYNn?(}@UGH#>7MZyd_5A*|m!l+7Ug?l%o(Hxd3dgij;;;5B zmJMYeK==o57|MPW;orvh4H6v9@o0;*qN-dni~;-aJ!r*UD4%)bh_dn0%N?i;ZCicT-Etn@8{ z63IN%Oal(Z0bO43`2|@I19bc3NvdqwiH9op^M}J(aiG$tHVF!`I6nx*7Qf%Et-vG- zET@u=vny5HH4=Rn457-% z) zgr_68>mItUKtW52al-HbSU@hMeFhr>Ix1w)u1c>qRzUA?Z8VdmMkZskXtwrsz6?aT z2~NDowz|cqy-u4Q^lq%V!tABCe1v}po9MfK0(Lam;BK}cf8zIXJZJG-#500t0?$02 z+jv&+Y~X3*c|V>H;-QVM{o{DY^FchcNz-0UTOQ*+gy%&(AH}n-2qZpQ(4jdCaoiNVGv{3v`X4?7r^OHl_EC2LrHu!&DWpgu}g##~#%J4%E zRfcEhFI1+A>ny~7!kg1X#SKKDN5cL4rHuTaSAzZL%)J`xp}xKH%Nm{>wmdI*_~p0Q zFAFxaR<@iq;P(SLIP1@_LWuvM+u?#Sh{ZE{A1~ggC+B8HhcjGv(hs>gs|+kxSj4?< zK;z3j-;#~|&7th~=B{Kvh=<{bh5r6L-ue733z;TP&thoT^3eO`p~mO`7W-pgUhouO zmcMVzJoIurETj+lJpaJ5|0)Y#8S--V>pMKr3+emke@)*%Z{zU_!&koi<^MqBK=_U4 z_50JKZ`C5+pzp6#*v`i}#&6BCFMlh3f3tph6U=vCr$4jr#BW1`?5{?@e+<8oP5yp7 z`u!8p?+-=4e=_?0SEAqlZS?z7_vB^%V)Xm%s0C$-uJ|^D)EhtaVXQA8d<7xnuKm=Q z_@I3hA-nlSgx`d4@UOz=i}0fe*}X3!d>7;YyRq!s5PlNjUqG0>IF@}of-QvKf$;MP zzYF2p?BFC)B( za5j55JBj~JXJ3!Doql-NLNYw$&2xm&1n8Tr`)mR`Yo6JNKpO~kVHR#dW=`{=7Em!;i7 zkD90lR&Fbu{m%>@rvMR$$6~f5?P+Iy*)gqd%mXB9?`RmBPh&uNxCm$b{Ls)JgX9cZl;NOO{T4BhsXdydSJ(Z4`t<)(VsKfsW5&+fEs7XCU8&3otRiw5?jkQRp23` z^=<+pI`;gsh;|T%Ip&llj-YMpD2dd8<#DV?8(16T7V$v47MjZM6;NkVK&5r0svJqp zp%~O3#?oPt(`kpn{&ZC5Jn@8mVa-ZmH%oBF>p8OzytdIJmNuEqW#icdeo?C8tmQl) zUD?DhLP_C4n#mlSLIHO1ZAE&{){w`LHgMkDkrMK<_`MsRetBXC3)l z4OFv=kZpIoATE$2lF||rR`IWM1Mit)iNooHz=^n5lkp&}?gK4a##>YPOD7e{;YkeY zgcRv!8@abp3h65GN&n4rj-dPvjMgIRtpit!b1bK){7(3@R6>E+tGqv^p348$q_h)~ zf<3u|@|%eB`JQv^k!%{Z4*|MUNY}}Gh(%M`)zWWR0YL6K{MVV&`@!C`l{H`(aWJo; zB8+3U@O>2RRPJ{l=IxPeM&8!&?;=WouN`t-#3(V1$13c#eAf019#Y79Chi)5D*xlY z16k6QdMDAMWhsTcZ51u7WKSTDSfgBHIC}{HRX}0V9!wzBlC-BOQe2i*A5w$QDDuW3EMEjb)=X z#I<-i_08bs$qs2-N}YwmS|T;;c49- z-MImMOy1|PQ)~;Ko>hrMLGxn1n4=rs)os|5p`BMPt$x~qkMlg%Mxc6UA$=5iW3kpA zYcvaqhhZC_J7y~zrax5SEBmJH$v?0h#wT@HWT6a+9iu;)X`8$Pw0dQGb(M?Q_#@CJ zYB-&-xqAcdMBbQLai2vB0CRY|$qS~efJSr5HwNI_L}QJx4BvR36&|TT4%7&s2``qRGjcj?jmB~>+=s&|#4|Amvzp)hvclhvUw)3;} zOT?Nko?pWAX*^%R^Cdj*{5h=E;<<=t2G0tf590X!3|Ai{Y(>v%qZ=S4jHKGjeU z3buV3=6yj2adf~79IEJ|g*37wQv5n#Kn(l$k;lJnnSo@Sem%|Yt$5Fam~oJHoR@+Q zsHdaX>iJjT&Vkhgln&b-J?fEu2&+ImZVe_OcfHR>u~rG^3}TcQkzIk!P1@#X!eFrr z<;uCBFyA$nQ5>;$7Ki?9H@RkE?#{K1U943u-hscl@85~+Q7t&3h@EbRkWe50m1@04 z_a>>+TYA0zOucc}+=B5f)T5osh)`H8LQh)R_$C?z&~hP>>#=G7`Hs!05roHaj7{_k z7{X46lh=Hi4Mk-0zJ5XP9D#(ajHx?O`FE6qb1BdgtfaxtHa;*-tEKFCVC$49NV^60 zs11-{TL}{w5m&L92g|oIXrWE+<2nnQ=7~lfz?D(yYzO;}@wn1#Jc9;r)v#r0q#@qw zVT}(k1l`e**V+@f!r2Pz*_FjR4Xhb*JlIw)ZN|8Q_A(V9n2+$7Z@Qr`PmN|qF0)7x znUgqd+M!iaywb+8jcg4DORlfM$g19msPARWpU>d=9G)-Wc^|m;iR|6NL%$!=?Fr}x zQ%Ls=xciChJ>cSSCzkm63~Vc#+pcGmAiVS7eK%13{Q5TNdo7!TvnamF>NdVhh%_z! zKsvZn>cThDH|;0$NhI=a!;mlQBi_(EC=Esyd`9;anDqZqHpb9sH# z*Ydb`qfEAS45e{_l@c~(4Xpf1$al8t-PSjTv^NhSTthgAd|L>oAtkaeQ|LR_HbxLn zAxg`qtf*3;)!t?B8odpw6Vs~2_G^0@=##!{ zyHtPE7;j@5Hv|aqZ^pt&gad%r3jW=-`uWEHjzj9@cm79xP-B=%Q(nuiK@#5-d7RoK z|EPmFb z_kYmzU;MUVXc)uU0w6Pi|M;ix!+-;0hJbscN&I~FB>EV~O(N|SV&;*07GJ%LDYSMH z@w3oyhNCj(@y>Ytd&50%y!TCSe#=|mcI4~c{`KGRjqmuTKljb={PW-Pt?&A_zwqtf z@txoG-Ie=#?tkFu_w@E1d+?#-C;A6Yo;rQz?74>@IsfQ|$1YyF{P^I|m0{d+boJW! zyWcZ0IW;}=y|Z)k*Ppm?^U3eKb^EFJE?|ES^1cP$*U@a-B7e(nz>-OC@d@L3ICcwEEm8#EmJ zfBvnu=kI9v!t)kRX*gK1_y72BwBFy)F#Bl>H#K~D$ii>Y@P$A4*INItYWTuWYnVN2 z@z?DAK70SqU(|N|6AfSfguVYI4PRKa@S?@P%fdhWeaScYTNeHu3x7nzm)9&DwfF}# z%-(3>@BE(j>sK@!{26=yAq}%74PU-u?+@7bH)%Nd`@gICe^bNk6B@q!AkS@<{6TwfS@>QH$1Oao z;mhA`?_Z~3_MiTxl%M@83;(f(gMVMcm;aW%f7rhNkcD*%Z`t>28oqqa-tV{X-(=yJ zKd<%tI}Hc_xrQ(N=Ksgumq%08#eX03P-IM#j1fghkuKkxiljoJR5COwluA^}MJb}B z!8}zejc5>E_gorC%9uH0rp)u4cb|Jt&-1*k-?P@c-nHJp-sdcQeD-J0`@8qq=j^-p zCEeeU{64AQBJl!=CrET;Bh7-1*cy^+kh(1CE@&p9R z@-jnxHX^>9cI9U>G|yjlGfZj=@}K0`oAd(K?UYpS5!FC^)a=my*YF&Yg8MHJ+;%yA zZaR4X;_C9XSJcO-%<}Y)Z()bhe#5pjls>Jd;rMX|?7p*HAxRy@&lL784`Ko@=S$HK zYgZt?T<=;<7Hm4{@;Jx_`KLZtKC|Qy82j1>Y}e=MbMnFe%6AE zsD68!JZ3y8g1hIrXNZJ>K9*b@lDM`QY)jkQ+@p|ge&gx(>S8d+4pbP@9;2X7E;Dwx zl|b!alVQn`w?g3IKDX+o5dK2|s z;4Nwo5ys*nhgZj8O%C`Hdl(9;m#92Fm+4 zRD-=ttiZ-ssC*)x!siRCLAF3>L3W@L;;tF?HZ}0e=kx8%m#Fx~ZF%lK zU50c8*@%PRf!n?9wt3ea)V|X~jh)}&QDE0n=SJkecg52Vw_0epb+uk#HV3MobtNx` zYhhesw%V>2R8+o`vD#wYE)kZ=CiZvA#tl{$5tuS9<&rCevSs%QOpBDN1GrUi zT&y)j_9Y^ldx+8qXA)JepPY%(r|pbaIMWFCB~mqhCT~LJS+OAIS{1%6+{|o<+b9!D zZhFQmS%uSrQmb!Fugk**L(C@@{56j$YL zTTD4DiTv~a@QW$It+}(-&h|#}AA2<_xwHh=43(~LbQQ|Pdav^@$SB6O?&~}A-O@qj zi+|d0QjDA3e)UVhQx$~|ai&%l;abcG2IWSme92}dE1Zh(dl!PWKU&D4^g&Nxq7YA9 zD}X(DvINCn6|^I$5KpM{G+Uap7KI-Sl9w*T*EEag9(SUk`fxk{$gTj-+WJBEl%zhY zKYfAo{$FwCRXQVBhyd!e!$)V=VH*L^B`CDxn)j-?^b&redDaeX6OGEE{16OSSVx6&1c~D$j*&P@;s}YuBo2`{Na6sA z{Ur90*h^v$iQOc2k=RLM2Z=vPY$vge#8whpNNgtY2Z>E2Hj>ytVm*m!Fr6iWHkycD%5s8H)7LfRr#4jY~llYm$JQ6>Vm`h>~i62S) zK;nB6-;tP2Vip@Q5qA4CXOYi|BvMJlNMs6=I*C*gF%p?Vq)sB0M2tk{Oj0M2N+L!g za|WrCNF@;?kts;(BvMJlNMs6-I*C*gF%p^lq)sB0M2tixAE}c_B@rW$$xG@aQc1)} zWb%+YiBu9X5}DkjP9l{=j6^0Esgp=05hIbwN$Mn0NyJEGa*#TSR1z@~ndIjdX(Uqr z@j1r-X;fgl(Y#d5<-p28V^}~yP9M3|FIlb)(t}#{K^6jtGwtIT06ME@Zl`}j^C8Us zu1?~5@PDrCfGtAvDUA7f^AS@hQ;ns|IHLI*rZM|dxH+iuKV8~+2F+(Nk$s$A+kj!3 zWT{+-=FezCxz)}ZIwc�(#N>2s^eX!_Wo-OUHi=u2({wDYQyyH+T=$UHZIb6`|LE z6|jT4r|TWHY}X(@+L1YI4-3T7z8z37LA-uE?vNvxX@y8_uwRaNd`xiFUf8b{mgewF z6Y<_s|4sWri{XoNZbkD?EIi=~;{dD~UOq3_4$X%#DtENTA=pLt({kExhIsN++8rmT zU7&e9o`=Y9{M9;%BXEIs_rlk^YKSw_#4b4l{l{4Tb@ymCE_ovQxZA}OO<*feQSvHlT%K9 zJPFPFi(YQIN+tYn%k_4LJ;&gvb|~?D!v8^T)hXx}-QJc2M1ClIKK-X)iN8XKu?>-* z4f9$adw?6?T3Wz6BK+lF=KDQCLu`gpbtsyjVmn@DEj$e$IA6Wj#Uv4zpy*bd205w4 zXZ62qKwS3r%j;(#W8YF^nJh!ZOS4y+o`vF={-5gwi1=@2Gq}CLgR%UwW-DP|FK=yP zcmY1TM`VW@(f;9-dXICUu^=Q??jlkD@oA?u&x4oz`{xrDggq(p&Fwf383RWG4)78A zyRd|D&l{qpG93M^2>S~O)m}ye)nHS*&lFX}DUoF-Y4D9VKiG2yVIRU;DMd8!uqm)! zv}Ot7lP|BT`+%p+k`p@FME&Y29KGTLS^G0S3v#O6@00}p|Ra=>Oeo#MOR&W6lIu`1O zY(w*Z?6Pv{(u<&zG5T;vyaM9djM9LMur{yeNM|;oI~rEBT!i=iPu5ExBkIHOW$jvj zP=2iCIrj*0!Ud;9Tf9Hyzu9gTHIJ~LKj#bi*Ylp9!cEgY?ut)9^Cv9YxWM8RzF+Z* z^Z0yXzEOX*c*q?uJRMtlZE^$RK?h4C+;Oj+#k(n7H9bTT8yT@)Q0` zxej+7!?_>Mnt0exaIeqXuOlv$`;57am_ODltXpvmH=BN3uxvTOd*|*+J&H^C&NN{> zlSABINORm3=a#s$E@dr|KYn=*LsvY&eXighIf55%{~F+ee;;G0seM87VJ!Svc#$)H zVav;o{$s>^&s@oVt}`AwLKDr7K=XU7jiKmp1W$T#`N5WCQN(pUTOJ<9Q+Ek-3#p;` zE~dLRvDXPd9r*10qlHBM=5BJ(b;5UDt9)|Ch?ox+9-K!zgdd>{lu&hv_SkT@Fy|nC zPRFk*v~v;S{L_B34&p8cOEr}q6Z3uStoHT;_^+7PCgM}Xe0GjhW9)wXT)Nr4jE8Fw z*DVQZ*@q99v?V28}S{sX`3AIFDo~Hh&_&;m$1Te^PBc~-c<$!kVYScD^8LH;%irHV zIAVaFCox%tje|D0gc3hbjT(_Z^Q$}kcHl0*Dzq<}5YH1-o&93l@dbHiQ}K6}5%EbM zPO!qGUYbNWPZ9BjtZ~w?#C^8A&3NQaJFuf-6MpARP=%om(Z2D*(!(bBF>U!pSH=kYw_jE!Zj6`c z4GF&6PQ-7Y>!-O6UzW}z9xO-b)XGI>Yw+~+HT|pP37z)Nvyy_ltNvUmzps#vc=lBZ z9h}RP-}pxzt31zx%{1}UXtTuytI_irMicFjP{uFnZv9?ylPC|idak4t{!8_WmTid? z!T-Fd#*!n%drRheHd5EI5nIbf+8UBmNp8SKsy-W;t4Xd$a$Ppkbl8ZkB)K-pmy=wR zjZ_UbGSx}0M)Ds>$=OZ~U8!;X>(zr;@Npg(j(-Q3Vm|`P! zlH}tgA7djn%0}8S$%jZjz(#668=1W%?;&{?$vfGIb+D1vPVzRAx3H1g%tmGt$s0*t z&qi7u8?joF*O0uL`Nb-9mzeDl_He&H?q{WgvhU8Ifq(-ul8BX#rlHXz@?Is(s8zjF@ z@(_~WW!oL|4jZW?(h^Abc+xIOq{gv1Etb?{NS#D#H0d5ix|2waWOG^ssfUv~iPSJs zzfHQ6NWDe6-((~62C0)s4JGyKq#i=*B+{;t?pH~75~;y#P75OSE2K^$HIUQ;NOuyc zmr3_aY-IYAI*HVaq<(?a{YafenlI_@L%NelrLj59o7B&fI*HVCq`MdCP9pUzo72va z`e{-pk?Kk69;AMXjZ_k8?xg!kHew`FPp~=dIH|jlI*HU{r2A3QokXfDo6}rK-I>%$ zq#hyl!=yWjR43B?5F42XNu5OM0aD*j>ibBYMA}}`-H~)Bk?O$aG<#CFBXts~dq~}u zbSIIzn{?mBM&?dZCy{DH>N`k%JE@aMvnJiGNOuycmTXSjM(SHhokXey>26NClSnmV zbJ`YC-%RQxQa6#hDXDK{Bb7v&3F*FpjTni)@Be?!3x0DZhP^ScX^+Bs<%#DVES>|) zHBhg4?YzB(cs?R{k-_2WD@l%g#PbZX2d~^9B4>}urg_Bk3YN@`eK)xKWwFx*CF1!9 zJH$n6FgV!qJi}n@%)3m;bZn2 z2u#cnWt`nia7=(YXy7mMI`Z=b@w|lD3o?fc979IxZb}o+uUPmDY{cLW|H*yrZ&n^hpiOYmPasT>d}B3mZv z!YaR#FvbbNZ>&!GS|}3wj#*4jFn2P%dUg*h{csT)7Z{%^*BQI5Md&|7sodc9%(-FM zWeLGK#p}62?w1ql z!l$HT4G~^dhA9Z4^NcN>p0oHJS^F6fU>f)B%0pIrAD5S%2`%f6rP{lR6aF_T*3X2= z{_~%P%USiMtrRZ=sX}iXcv@KgOO>g@P|#5-bs>gjKcy}8d2Vm4I-+_rNSxt=ASudzC*5E1A(xK=jSiFkj44bSNnfjOc(DosOK@k`7z z7X{wfiFPT53ohXo>HAe406sQ9ue!S_hpi$RYakJTp`R(suC z$P|O@)!vqY4J!$~P)1f9N+U%-_J3|C@+T@s69>2O?Wzh~EPIrfFBFG_!|8&)yIK60 zf`$Zmndt0P7-qHCJH-$Q*poiL#bMn>B0g!QUJ2;UHdv~x$;z*xGF1}9ee`0NY+>2& zgT?WZK+`<)G=s|WFRqHshAZJIeZNv!{pI2k``O@e!%79E9&QfkMPs|4=$kfbqqV8L!Ap=yUzNaS7p_w@T&N^@GEg{ zOJzE%Jx}WO&IM!Jvc{9_bbyoj&*X5Uof(u&tnfY{s{n(_ z(P0e_S?L8)HzCBqW1py8QHARFVG+Fb5m!^KCdMBcr@7}M@H@?rG3r+*xSB=$BG|oZ=KIx>OA*JywiYe|t?X4h zC-<@JhuacUg7Y?}UNS;_g#Qy(8cMMJ{oBm;#jNr^wBDfv16#8@158-?UAW^i(xsWR z1jJeP8)oxH3AipC%vBa6@`s7BRdwY;y@d8-IhxL^X&nCb&T9wb}TG1svP%WfAM0*t=4@**IOm;QhQZmVd8r z%Vy&l4`+^j-M5+0waca@@q^hZ;^lLR_f6Q(vI0q5D_PpDDq54^vE^}+xSPYg9P^~L z1h=R-Dv76KU%MBrUPW-uN|40GTJ5fAJZ9-0l`@jJ_Q6FfTN7F7DOL4I;G09ES0A}d zCH$9E;S#v+BhBeG7YqsRU42ag*L?3j+wijl!BuMPB=F$3&t}+6jM5P=s#zw158SwV zcd3jXp?iLx7RQU!zxnOSRwKA{ZGkvG;1l2T`Wx}S2@9-!E{^lz{iQJ#CInZj^A^Wn zitE|xn-TA0u&;Gy;<$o`-Qmh>tn^*$WyJBs5dXzX*2@z5%!VE@TkPRz8HH~&+xWud#<$YqtgnqJVnHb)%cKqrkMOJzh zO|!)C-Z6#l>8BeA-Q`D}D1KY5Q?TBLRsS5#&qeVR>xiE3uB`gG(d;dXTUd$g89vEM zPqD>J6c-gw<9&CbpQw-278z0e9iywxDW4Ud+S((6n?BO*_FAn)#Gl`ai{R9(#Z%mC zSoLY$c1;Aob-!%oJSmnvq_iCs!N2hSd=hWTYOhu8Ac8Bow~VKxs1xDw_E{o0*LkDK z6M8KF{-yXVJWp7li|4^cLJ#_8Hw(X7yGiJa39EiX%BF>J{Y$0AOnh|`m#(O5NAtZZ8FOJ3AzRS$ScKx&8$ z(M}NOw_KigXV8@3^^Ma4_>Il#DP?z9?fJSXP5`HlrrAw>WVKh|4-mj-Yew%rWYtWR zXGe1#KmM-uPTjFdR{JQmc=O{Ib2h#S^JJB;w?&2@FE4)bao&RkMEtK?k+pi(OD{V# zQ-|QXZQgwNrkI>lwsY1Ke7sGD509`8|333AEBu>woEPVKVaUro#mfKjpLV=>>)wQ+ zmK0WcsvXlj`0W)+{ByEc{ja_wjt6%u5*w>4VucUw1Rh*(-u11|5?SL3wX2RBuelnh zIE#mPpN(~Pd2{2!rlH?CiU1C?2qvL@L}1vSpPH! zzDW7_gPDe`{NC@6CTo1# z-3h-K77tVTg>hE?EW5m?7~d(@>*^<1^|7uSpJZ$Tbx?z$bAsM?=`2{Bt zUv7{01mn%2TjgJ@SoZU<7awO>McnzA8_wz<7y72h7$-w6aZ%e?>F?-=F~*#MTesp1 zSoWzn;62LluMgN;Ho%I%a{wP)(0@{aqnKn z!yoNe1jd&V{Me9p5953d!^%dMRbO+4y}KEu?}|Q0rm@N+GBVxC@Vyu{ee4a(zDGyA zI~XqCbwwsLS@9K)PPa3Z?*!g&abe{@e$2a#5z+D^y0?mD4~NF5TNtM8-8LFb*7znn z;oZ#0k8^up_)~^RKX+ofiShJQyWxTTtp1=eIo-%uG&eOyD~DCS+)3{S#@9rt^^ap& z_qX`*={iQaeWJc_9qWD(KQUdy_&g!HUf)ZQh>tcoUCD5-6W@H9#z%1K)N~mm;@PSn zcbP2v@12@1VQ_pMf4eGcCCmS>=|aX4DT-i=C#yZMU(=r%{eElREVr`8BigU&9LC$x z#)}R6SoI-0J)On4FJ@f9Jjf~^b$U9DVRxudXb}%9KeXxT2aK7*SGR3@#Ts8w7XH3w zzj@(U>ir(Nx9_`s|J`YWy)Un*8k@hu`PMiYtyM3>**@8yOCbJVv3w#)e|9gVeR&h9 z;t)k~pm`WXF6{+d{$1PrLfq&VyjFbCfs7`@lW0Jbg`w5B+CwknUJu9~_w#R-_Ob6==`+XsBZ>MQ)WMbICmAJV*3+6R<lhE-M|<-1qo{`E)tp|YJCu{_h4;5_|f*P0XJXQdw2#{iYx8&eJWc5nMx`!S&<*#o~S_T~lpo%o#$lUS^Q= zyt5zt6uzyWmFGs0Ow$zn#XA5$d^L*?5BgIyD>h9YlN$gp?m1UmqXQ|^v-hR)QwAW> zX5|Gr7BF1&_3fb3T`j=Do`1_A z6wrFO&qQsYY{#U#)*K#$Q-S-6Om9Y03MpBy?)VQvuVcvTyIChFwI$`fs&@y$no%_1 z_&I{2th(}B6+Q@!hMmvD)}EsXeV>X*tr>*no`>nxo#B)ROJh!@jtxT7cAH{>7dPpK z2MX>~%^rg88<|cQPhF$?n#EWz(H?>)loV;k85(_7sBPXoiy????&^s?6ilyDezns0 zDAI38b1Z!uL7B=4D-{bKf>!CsT@AN4gI*NWGX;Z0{&oJyW7+LpB*PilawfBM8 z_+hZxmnyFPJ&KN3&2SfE4#UhD(+8~sj?UU>~<-;AJC$EzAPKN$rjQ-ABe z_S=-8b+2Y?6^z28rgp78-;U8&dPqh*>l+2QzOEk#zQD)21@WF@5Z6fxiwEa^cwwX$f<|)U1gNW63YSdA(u=c9Ll)*wz=c?~GllxT6oG~1yR!Qy;CS>A z$eW6#M{m9OJ!#UEWm=jh&`qzf(&6h(d zuUdI;x?G!tb-QUB=ug}zP9Z%tvT2i0ZPH%!naiCr{?68FM%^SFkMZ5Mx+0kF6~MPq zjc*E~&zxSXLyxBS2MFg+1#?DD0A`+b_-M4N)P zx~?l(3$N2JYE^T!Bu#XZon0O_IR(2+=a_v- zyiQk17?Lnj`USTo&iE+54xOV%#;5XN~sFD zcjx)NUvOh_w3I@0EWPjGRkaNTzaZ$T=*xi{(RAgV4&C#|e?ju=d;WttF_f^3BRWeK zO~cx&LU>}r4GML=>#kGg(-4?Yq1I3mL6KlYKb<-|4FW583&q~X&?`b>eX<@*!@?Vh z1KT!SqV&04TasNe4Xvm5-#*#lNMYaqb=ey!G2OMl&m;Ve|NHy^-N)IN_ixlZ7})Av zVFh?RcRBx&C*ZLqn~PM@_N!Aaq1s&jsEt>rpW&^=>z>YUGtzk{o(<)6d@{Go}Lz>j;vfeGkosw zttnu;)dLs%i(1%}gZNefbYea_O>!&x^|3HyyZ`>noSt5heMs(3)vQm`WDLE)&yCJ; zLBAOHO!n_@-{^Cj-kiVv8&KJG)kEg+cNxN&Z13SzHvX;)He>(&y)W#y);mqQlX1!Z z(=+iu4gDaEhWs%RG7+*aT3)}28~W|DH@qW+euHfWMzdvH6FErqL6yP!z8bdozwKd1 z!uO{9@gEdSvssWXpA1sUL;jPiKTxoD&y~JACPT#1g+~rfc2h99IUA?%B*S~({g#^D z4HPU!^R#?ZGK`Cy37V8@rC`z`Z;TXDz&dz#T((0e1yjrwbl#f+LU(VJdZyM>uwqyD z(7P$H_uHuz9di+Ho;#V|lmZ%IQ**b>L!41}r%@pl1hTT^`4uZDm}9xi%zdd4bnv;w z?2jltuQ?7n_fny)J>wdEehme4cuU*+BNc=+W^AyFNAK-LDhoptUx2NZv@FcZrC_>O ztnhs=!0X((fxGaHg01Kj>$&#=&M%4Lujxne&5;t4ZGHhN8V1h|pMIub50p1-UGx$j z&t>|UD7~Oyd-QTI?tcjea(P$9^4cia3Fqq>i7#RQySOlD8v3v@azwMdC;xw2iVN403Sd8*(B~jq)@)n#d1-jrmvM($GYpr%Y zr$br5{8HX%)L)ca@4f4x!(n$h?Kihj{%1GmscB?FnbgCPJ3Yw0EF|+Ij%5N*MzcR2 z*hs;iy?Vlzk_neMUx%Nq=|J|!hbrAv*l5HI(4tC*8Koj z0LW!#qJ4;zTOYGde*ohr4M!I?^&c51yXa@zWpS z{sN7}(A!xQEIvQ~IW-3oB!pTurBV8g%ZK0lJIa)Q2cT!>+7>~Ajag@!^?wc zd@8E@!oi&jV`Vu4X-!2GjCnXgWqmG~N&I^GBmCG*qV|2dLtx`4kmEBq(^)l1!G0~nEv|fmq1Dpu zJMN+KY_x5R&;JD0di(WC`}&b?@oiWr4|=w{Jgl)ttW5Vf4*}@ z^*0c9K5ei7j;ed6)<`0K(Q@8>^9v!ye^fm4ET)fX%RU|5R0yJD>LPIHu+J|mNA_~pZq9S*Vvuq8F7RwWpFVaA zH?p@c2C-%1b7vnQ(o^72x?2plR%~;z)JFcd``^V{iec-kGt$p{(kYm1QDE=l5@@WI zNC|zjz4TIeu;|*Z!BzbFSfl9k{^?RU zvQ?zSC>Y(p3sA|^#0l0su|xBn6%ViNyz~un69SI#s3QM?1*6M9e}iDvC7w=; z(fupMxJ_wR8RX?07j!<&t&i#b$XULv49sC^6CdoO8Kz~0SYP9HU%rH@&U6`b&>0A16CX;&1{c*OnS z#yNTgtZn6TemITnyCD5VB1a|kUAwKub*~!T@2(|v8dbulTS1Q-&Z6)?0NIDX&KDtjOANhcWLpLFm|cdKBP_Du=qdNkhzA0vxc`9rUt)5^q|Ggqd!pnyLR5srPP9^nxM(B zXAK2A5Gjy(p%!>dp1!tyfbx6sqgi!+Ep&aZ@?3ZsmH&l;jG|;6Tygb26HtNr+f5bO zaN9a~;2Y6$=mWBMT=|#I{W>^pdx$x276R?9gwxYelX#8_L$2&d09{3_& zDONp3`s!yzq11XXTX!}6y2^ZgEN-e-e}6sbStQ%8+l|Jfro1dHupW$DH+)%RgXRwl zz=jvlBTiYyJ>Y=mu#aTvQADZ88!lN}Bpf5Dds!$r;AI0}IY&qKi z>2J&SmTIE%9GHqb^05J&%wGC)s7UBzi7#A+MH?Y_q2Q<9t;pV-ch&dXHp0Su#ShU= zBKnxevKRXvH3DN)`psHDl)uQupUg%Y!QGdx`QtmmFD`qtt_gU=WHaU}N$X>2mt^mR zHi6TxwVE?*QGV7t@rpGx!H)a-5(i5VH@|s)rp6Dj;alC_*Np04TGsZq_YYVYcsI#q z3bo(7nB%+oJ*m)z>ccZ&4fRkf3|*Z6wy+lU_nj^#NsLzD={suPK}C9U zmCaAdHfZ!tDZshV_`}C>VWw*vxPK`OhZm?m7KI*}o6`n6*Kbx+-h=WZvd(foIxk0W zXLg>e{ybDaLmJXv?Z8)PRy+9s?E{;7^-{H>9bC?t@D-m$@q3T%H(&7+mW-dW@#NyA zo}#YEo+G0rL z2(?ev5AQ?!r?1Y)-uVXYpI$7oAYB0Ac9SonHfSHU#N3zr?xTIw6=$EN^r3y!Ui^`x zdI+U@x2^I;`>AWE-kium`>8p@5+4X649*xTHAVZXf#c#tAlg^$9HTxz7wxMq-}z-Z z7s6|?tJhP|{_4B~+ssa&{nhU`c3V6|`>R9GG;gX#`>VNeQR+;DXHT0hS&jBtzq@-_ z_z>D>-BV!H9*Xu^t1V7@`xfo9zT+Btu@UXFrcOOG;zJnsq4Adr+HYNA&h>0#K@Y6O zo3}Zk{np;OdwMUT{nnCi4gKQLerqSM{WH_ierx~Bkyi@Pe(RCGm)H-q-};urr6a>= zzxCePKR)v!+|e8@Cz;a&x~u%0<WCqKJ9_NM_Rs0k5PVx$5s_S>;c2M z{mCEAaOyJ0r>KM;c;dVHuwq;fw8k&bx)+V&JwgAxAhHK$I<{X53hM#u zxR(P1H+z78k*LtNP{fmd7A0Nlfx6p^sG~v1ojcCTAg~9t9xWB|y3_-OHPJ397mz>N z>m#*3J;2Mcil56H*{?|rRmQ6aveM1Pm!Ix|papQw;8YLXj@{t3<^;0$_q@WCV<z+faGo+(K>h9%!tnwl3O)>_Q}y^MFYYxQlF)>0giR5zk3-H|zn; zmz20(gC2-k{CU!GHEQ23KRjQj2QJ$l{h_Aa1H$IPhx|2Aea!r}p=@anMCh&Xl~(Bi zlgb;SyBDGMUHWz4x?B(ZOn$Qnm+pZr!ZP=o=JtRh|NA4;l085>>Tp_2tOryj9)4ON z+yl|Cz-FOf4}2N;s3^sY`W2Kc=Hu*v?$5mr-BaB#y6W_cHzVD!N>XUm)}C%y6xzBq zx1}45p5JYnSBqBm)2UxfzIDUlIkJ70=zO*pTwEHIcikYP#eaA@y&JNQC7q8=?gpu) z>6_+0=mu)5UFy-8ZqP}X8=P>X8{(8-q(2Mjh8xz#5@Fum(7KM|xzoKH>~~X{qldde z#n$+vn|(LT*mP;{2di#)w$c6Az{YM^Ym@zc0=l6C+o4md-3{|wEMp^8y1~>?pg;qi z2dA`dASPa{8=S4)*0%F@L)v|f#^2KgV|v^dY8tzMS{RK7lyyN|_a@(2 zd0n8;w?EP)vkPR@jK1AX?t+1JmeVPByWs2g@uslbU7#wIu-Pi03-ZrA;`w>D3ugUr zF12#)0znHkpK$vwFqoWMoU*kGtZ%IgN?6wgN4H0xIJmM4tom9vaxLzHIR<#i$+=yi zm4D&uTfr{y(X<;cpXdZdRV~R+o#=i+3#-2Ry%V;)E$3JH+6j@Njn-j|PHd6gJgEbrP0l9cE&Kif|5s*7)Xy$OX|dE@ol z)t#_X^+V@%)lSIt`nF;fx=-bCCF-T{b;1UJ(>r5B9dJTtrkL1|4(R)AqBdUC0ltc7 zZl&NI(6ws!)wa7GaJ9_9OZ|KY@HDDjf3&>=LiX%FVWZIjdRvd()#2{|mjuT%YwCW2 z@wm3~iB~_tr1B~!?SkxSFN9rYxeM_JI_xjbIHn3FK>t0zczmy ze9#Voms(x3J=($4^r3R7Q9JkyIM3T7f$ryz6Xj(a+Mq8iDW?dXd*@$uZl$wF8%Rq} z>$OqQId?icbqxgCK=ERx>;ZJn-F{^i=k2#!;mkU!u)wZXxcOl8u-n2`C^N@4`q#IB zQF>A?C7}iKEtEF~*tdXZW$Avqg)N{eyQUqqiNu)eN2<^TV%iX@(;|3Kge$ znnBQS+p?s$KOpn&X-}`CKfpY8rR+YrA0WE(^l_7-CU`w@=drkV6I`1rNqwN&1cH`T zIjtp)&{s1c_X3@7R3TG%ws~PA#5~iuY|U(d%evAgm3tcCY1{KPtH8@ zZ8X+NQK|=qvM7(cYaOJd_^L=;ss)$f`%$NFe}{3d^eB2%4Ll0=D#^W74NK)2trmV& z(6KCBINPxj%&nJC$fD;MKE}3y?(Q-WI@Z|LcIz8h`#5AglP`sgS(eTVt`&p1TgTxA z#f8BCYetbx#upGhCenLv=_iN~bCxd)cn5oJUIYw9zDDmmC6+7QzfNI)Px=_i6X@jk zrZd@k0;vm7+3(MPuV1j(?{$ygh~CQowOT=XN2koNt`GQf*7X8UbE$sqQhgt9zjH-B zqwO}3?vUfQ0R6-P=^xBG-*4%Be;r8wpm-deGXL-rL`D!-5U|%C$n;Rc>;!**-sAWB zf(XW5r}3wvkAr7wIr7gW{h>94e_p*hZ1&&!z<=ul|E&-Fw?6RS`oMqd1OKfL{I@>v z-}=CR>jVF-5B#@2@Zb8tf9nJPtq=USKJefAz<=ul|E&-Fw?6RS`oMqd1OI=m50G!m zY5#a@{`1H!U;y z@dyr?N^napc!o3P1?)?ZNd#TZZ&uP$5Ai#97Hp6+4ujAAmleJ$Cg6#0>NUJX4nU@9 z%2sPvf4swz$iUy}pEr>RA=007FpQP{tPO^DSEKZWQ}FKhD1H9mS!dg$p20`YQNOat z7dQte_wTI#1AB2c`MpjeOJ(!lSV5LIodYH`)Gde@xVBF98Nm8YLQWM zieS-^hS;1aL(CiPG8sbu|Ni~W&mni>_W_B7-{1aQkQ=)l{%`h@ka_supcgb~ zPvA>ms?hICr4!|;pQCI*oJ)YNUYKx_;NX3F1D|oh`-u6rA3tlEIuxMZt z>wB(b5Az-ROV{d|j5zI^2kd>0tbItb+TX8`>DR*!55=4L%d=pOl;jZ_o%Ow8vJD@6 z|I#l7wM4!+@dWNB2&){3Vts#Ey~gk=GX8ql5*^~bbYl#jl1b}J)n$Ee8ZD*%)L-G< zcdp;>j=m4WC|IlDNC@lu*JN9^pZqJl+US#0n(_CbO6{ZdnmE??xYeiFUn2EF_;%1n zSVsB@p1gacAk2~VeE_mjj_!Zy=F4nm?N5lpl@=A<4Mv~CL6t@wK6jZ+pGok4T)*x4 zZznQ5{7ZP8OdtLweVoic{xA8*$@0U$lpiPS5B{b8aI*gKf2n_*Y(M;8+7Bn&AODy3 z$4UEvf3Y8&v_JS4`@>25g@3VMoV0)V7yHM_{saHge{iz@!N2rBob12wFZ~z)yQTl9 z|9bxnPxvSLKWSgRf2L39MF0Ow|0Dl|PW1o3^gqhaiXZj=zw|%qkI;$!Pe+KZ-aqS~ zrKA4;m;Oilv2@h`|I+_xf0mB=Kbe00pZ3GjQUCu-|HJ+Wo#_8${Plm@FQF6t|F7_W z*gq?N)c^kq|408}#gFqyG{*(f`TxnFRmG_4~f|x6>cv$DiT< z9Dn{yAO0A>2%Q+ee&-MV82?z||55%w$4^54XZ^t+<1Z`zf7Jh<<2Nh*f3zR`G5)jk zf3!dRasOcH|FEAw?>{X4ANB`-+`m}*KkOI&xc?D4asMORrx*UXe-ipX`_G^EUqb(9 z|NHa)&5HjY{rC6hLCY@dJatYy)^ zKM%uJHOThWp#1V(vfo&8JM}lBy4?Nt83Jb5l4le497~1ze%9&q?9H?3U>njC4j)*yo~fV8QS1 zuUm68kitF(g%i8^@L|)`)!XE`C!AP+t4#P3V;XtR2`8rcSiq%W#P@%(_a<;XZEyeg zPDK$ilMsatMUMGM>#NCBgpj#WMbl{@P6(Nf%pr3eLkGuv%%^YWnGPNE5aMJG4Q9{h zT5Ii8bpF5l|9kG&eLv6Zw|iZ^*Sgl)YhQa!d#%0qcWqm-u1P6nXIZ20No{A^igip% zDZ4jqbB%JHEY~g3Rl2>oG^G8lIdYv6UFExRHP6O3Uu}DuuS=q<%p5v1;=79*Z1ea! zB)UqlOOp*yGFu`tw&bbl#h6`yF3rL{F)_`LO3R9|v2pu85v8MtAk~yJth> zIwE??wyk3ioS!>Nt{bAK7&ft7c6M@-?RvgWh@MjB?~RAH8LqRP$kzqYS7u-NSX00C z0$Z^Th`wUzZhG&o&s@3ghrZG})wk)x?-S%YANoq!`0OQdm6L4a__`jYmC`pOlxf*Z zY%TaY9;KBZ`zDolowdaFEMK>yv@*5x`KkMwuCr~%*Xby&Y$_jWHs9%7@l2?3topRbwlZH= z!$7&CbFymHdJAoz@^v%}lwFUr4>jL2ORk$?pg0bDJz#R)Pqt#63-g++vOGJHAduIc1Cf z!tKo}t+sXM>r#|c&fOYT_R8l-Td@vBIc2BOR+}v$5w>F82_t3m0K-4$WW>pJCXAGT zTJwXfM*7QjC5)7vOYXEW3|nCP6JJN7yt1K8$MsWwiL@2#MwC}J&p-5++rZhjVx5Tc z_!?_B(})9!wnO>45apFi(|_>P+Z|~u)`6&?H0o+Nzj5e1+iQH?hYE^;*~U%>kIc3e z>pWCYo=^F+#jn0U%XJ+pDl5*-bL}u=w5?djp`vW}YXA9vp3jT^weWnA`X(FO)4u)n zNQQfP^ACX!^AV0=0b{~K@fq`BV{9NTAR)@IpiqBr9C=uXUv#9`@JM{7zF-sSSCsS8 zib_R(mt&5-N?da&%7g;86YD|P_@7wN|8+B5N5m%5`SM@Y3##Xk)i$O@h8de zvZ*wXewFj(cez!{?{ll)=~m?TxeI8|=Y{z6kK|SP9d4E3(yy|D^sC$~zQe7kl;7u8 zRLbvjD=HJkceoXm_R_CXexF-Wxm13STc!GaZk6xk_qJ84-{)58EyGo+-{)2-zt63x zl;7u8R65J=ajPsP{VEUhdA*{joFx4!?WJF3Y3WyaT7G9+rTTqtmHlM6%DU38@}c~` zw#seNuX3#Pt8|urmFoAoRm$&kD=K%$?`^A`B>gITOTS9>``jw!_qi37>i4--u9V-| zRv9V%D!WO)N;BzK`9^*}TV>3@{V3#fS_cXe+aaEs{-nQ0ZPPi<9-O?B zex)~#@7Alj;kFa(m(mX}#d_m+fEvA|rEaI#@z$<`{OhugN}6%M8F~0!eN(;tPX@I` zdP~pb&pOLO>dwx!e-3-hMvk4<70 z(wL)B;KKwP*qeA}i|Oz`*~}4Tj1*IA{0{QP`-$IOWJ^B$I&SRN8c6T%!F!J`vcq*| zRW`E0`FAeTs*&0b!9&AV+_ z&pLb6(K=~=u}eeW#c$Fy?x?(Rv8XxaGOP7+=9}i>mK_y?n@UaFD=c-Cd;Sfxmau0n zoBjyDn||fs|8^_R*Jj4gmlChC{>|%_8J>sy@zk}{>voMfj*Q=S@Nd{3pv^IVd5zW6 zysBn41M5Y{-q`eE$#qugmubV6XH~-Q-mAtecD%uEV6B%{BTybS8?QX~_y+5>Y}$+z=yJGZtE7-qu`N_?9SkQ?I#^vp*qr6V7o76Ca z#otZr5oFb%qjFO-xI#F7Prx98?)Sj& zEDxq!-DY!-{WQEwNa+vQ-}Y49m80*mG0R#2)8$FJH{UDJB|=$o&`zS0_WYNibuJ5Zco@xyX| zX4z!j7*;#KmvuXR^bgAI-7EUfWcoJmKN*fh{>Lu5y(KJx)ww&*DEdQtgxk$-c6})` z2@An@=h1#s2jpM1M1e;fOU5+Fq1z} z-<1=k=Gtv!Q!k~R?0>O7`hz3ZEx-Sb)pNY|=Ehj;pL98_lI1ow%5AIV%-8UHzIt-MY~uyH*z0xP=Qba04tpOBjnLoCR*hS2 ze&GoEr=)JPMvvUX#)c$S&eFy2zEd-Ml)Jo_O>*gR_JM-)(O5Lt)p9>;-QT}e*h}gk zHn{Izcz~VgG0I|S1^n(f<58(bj}NjlXPQm=?oX@>lN5AlmwgI*{1Myr zXKC4n(RTWSn9qrb?=F}2A*%sTm)ls}P+`tCNSVpzfv<{NG@hCM=k zOWM^Z^1Gv~RriC(49`=3e0pbb`Y7ua@X+4D4C8Yf!^%|Q z=X#G$F*A!-K^0n}erR-Ot_?ZOCJbu2Z}SuM-`Ty7?NQFKh>gB3fjKx{*xQ(g)6cN* zr053bpOIghX8Z4#KFebM=zFf(8Opz)^cC~Yvhd9h6P$NrorSqoyxvzj$I30*GVew! z^k>E&Hw{>Ijzt7rIcWY9?DHJoyJPk9EOSA#SAi>VJxD5b(`xB?Htpraxw%geJ~v^M zN4*P7nOo!Wldb5F!fw=yU2=g1EP9ncE&}~UnxWI6vT1BW!#?JNMk2qn-nWhIlE#i# z-nVbT2-L^q`u$o=OJiGJYsX#pf_-~O)cNsv8pAr6v${RP`6gdHckOc;i(T#Ec;YG$&na!2K=L+X!-7ulG$ zD>nzWM}Hq%Zq%W_E;6k_#gyBRe?WaWuwkdsCHDQ*-L;m?M|=KO|5d2}<6S~8*&j+~ zXI_UHlO=WDT@T)XyBK#^-Gtcw*Dm_;T zze;7G@T;6#TliJlTt3J5SDCt4_*J?M5Pp>zCc>}M^Zr@Bzsl^j!mlzcSol>c&4piO zZ2lR(ze;>kocjw%neol-Kkpy>=luiz{=$FWKlt|hE5)CGQ@bks{>y*fKM?O9sqar{ zkw1l>pB8`rg2D?we=VMW6#k$0557FV|IhmeMc@D8_2)nD9~8X*^Z)tx576F=Z10S% zrsw!BVnvjNJRJ`|?0dh_B>Db?7^fEsRx-v_tI#nxn2Gl+4t=N{fvTT?`kI8IdLAdkm5ZlL*%L@FJq6S^}`E3eYR?hsUN7Ozxz3rJs z234lBDP2E|I@RPd+i!4S^puz=RzJ#Ocinc|nNf`f0~Yp(V%A-pr%j%4nAr@{uQczz zFEjHQZfE%L7k2;pqw;rjd|2tzb?R+N-pbZk4~jp~W*=LZKfQ)((AiHWc_vIh)LQp}T^;S!Z?m+UV2xuE?;Rerp!9 zcOKt6^ty1B8D5+7`~B&2*kjYtI_}#Kv)Rr2z3x9}Iy2v3nOd>!36}kKkC{tOEPE8- zP=Di{-OQ)v49nfS#<1)I)-`6I+08s!^v%ePI>oAHu>qHIn-$pWcI>vUuKn5qJ5+n^ z`cz$TZ1#ZyeOjB1hB=pZ7x+Im-gVQh{dRU>+t{l^TmHq?WtBZYY3C-^s72K=o%Y{i z>6T50z3jb?1)IDYu&G)aTR84;gSbaYY~%4&cZ2$;F}Lrm<|N_^8CI`$rg|qFV|&+5 zu4%eEfoY~rW_v3fVK;pSw>B7YiFI6ZzK&~H@Ihf$HuKpuspo?%^{1EmK3 zI5>rwS+$wE$KX19*4nYV{iS_uo~CQ}KPp{gDIUXrKau%68})I`g88YJu)ba6>&8oV zu%>r(Tc*tTQ@&qyo%vUd-`#v>SV8`_<;(43z2jn=&I8@E6<>aNkHsl#tz(DXw+iB| z`YoXC+VC6fv}sz$eIFmNN|trj?+Cxf%Khfl?=99x+O)Gu+Ar}}Sk<@XwDVtPGMm5q zxO(D#zcP0wYq=zo88qK}ugUl6taWmm0e7+P&@$8G+u!fJ$n>XnezDT;E_>R_pw=Ff zyX`Oe@w~>e4>jM9pdV!ytEq=}Z z@M-mE+Oy{@$+$w99l5XB+PPP9jE(Rep4mN;Ox$zWrL!a2;TuHk*6=0`>i?d@#*hBB zK@EdfETmgf?>RMcSqo+3t+w@Fv6%GEjyf^9tj>hxbt^4+#pb@<(`2vL1NL&U+W2?D)a3PFL|9p|Gvw9_YI1gvC$;jjA8A!R}GVMdb?o17B@OxYxF#=8Ttj_qeyY ztn$?r1GjCgq^YSKiCmZaj@{3)nlSTI70o;M*`HcPd}On{%KICwuCDP2+vi$2L($xF zykjwJc`eQPka2+_imqnT%drk+{54d)NoYf2B=rax#~HI3+hJ8MM$TU)04Re!4n&Lh+#-{|pPvx0J{G^1hK>-}Z5k5*-x#}C}3ZPuWVb+z(&cD)j+*WVGo znjLg)c;NR-cT;9}czffm33I# zSDP7q?M}Vj33eW1M_nmh(*ehM+$8J3DXrII^U;CkKm4KE`+3pmJN6I7eX_##o-gxx zU$m)tfxVM{?6oc$k7OziAsI+B8%1-~mc0Z-{)Y{q2 zSsL}nRJ*l%Qr2#-+mR`$7rr~+^_bRUK{aj5pV}AL8}z*1wCv$u+uLtl;y91?CI$BD z+?>ARuwI!0dz&_ijOeESEM=NgoJW-=%e7;E4(QwHmn(LcU$%UngX_Tb#GA?OXJ}LQ z{+_&8D(#+?W|K^ z?@6^tm!lRrP;+)jPP8xJ&{##oO`N>T`UfJl&wnqHq6VqjKiphWm59X&b?ZGUM)mwfDv3u7%>p}As-^1QE zmnU@Ir#*P_@Zq;Zs}$I4zWw8?;lHcnsW0`{_l(KWS&sSz_J(%ccH`pM_jYPMvR-7r za^04{QeM0n(=gF|ruJIhpDjC8J!yArR;_J6EM3g{+`H5KQp~!P(3ZpdZ;bKM9&t4E zyxe1<-J6XWYn_jKv*kPHjQr((WQw7l%gx?zTWbC8e_r-f$H&fb<*gE9OcR4AleNO8RmL9hzzrfz5hbo2JnI28C z`zvH&&i%34@Cs4SYtKDmm+O!|uS)(R7F2h58^e$VDH9ijx@9%!rDf^CXYgSf0@+VTG6nkgB>zP48lSD8{Fzv-M2c?I^WdX*CzIbPPcJZvZ47hY6QZ`5$_c^OA{ z1kbYbucmWwL_e>u!Ur!LQ?JMM{&wYFv|2FwkEUORtLcdT{cC$={ubC*LjKyx{ADHN zubs?aRzm*T$^2y{MxV^ zx1c_hRDWgvP+)Hf{ezwCA6NDhCG`)o|1Ge$g#On~ z_P?xz{?|_SzpRA**G~4otc3oT$^N&%-je!Xd3`Iew}k7PoxHxW60UD{^7_U~xW3uR z>l-WK`o`q-t-#)ru5WVuP+)Hf;|C_k53GdogPj~duoA`(c5?i{N*F&dIesXxx1{ld z9G?}~Tf+E^$?+L0VSHvM$7igB@tK_*pRp3gXH1UI{!sg~lE!Cp{F|!AD`EVrhNqS= z{!NwR-_#Puzo~Nkn_9y7SF0XhQd7)d;Fa@>R8>*J{32D(FSI4hFH+_FLR-T8B2~^W zQcIX$XyyDu&NoY*Ue!TFUif78nOo3@1cn^w->v?a{nv~vEYEn)tqmGifP`DRJ; zH#t8pu(yQyu~yEHwI$4twQ_!}En$AFmGfh53G-vEoFB{l&1NOdkLCS`0((oi|Dcuk zAG9Uhf6&VN584v$KWOFs2W<)WAGGrRLqYzQbpJu#-zl)Sg!?;Md4ES+!u=hsyuYI@ z;r@)5`m2+7j-cY32PhZ3*|! zwDSI0LH?F>|4iPWEU>qP`;%IEe^Oh*{YkC7KdCL@{-jpkpVXFce^M*&PZro)()~&G z_iZXgzZNZjPk`TbX&m|Q+Z07o`efU*J>wOUYQlFTax${v@?NOpx)EW$gyBEA$)zZLqC#9lL)@eM`uq912ShoC2ln9!bn^;v_Vaf23hxva?&>wBTU1E5 zsGv?!{(bzydiq562p#3<=R@hb55$_(N*~2Z>8-R>zE_$lj*1KZXmj<8i170Fvk#5# zuXOf{?Bo*}ILfboV2E$%X!5&wMM#}nXjl}Cfv#_aUwBXN5q>_AT|0`f^m7gj@}pz>DbWzxnkhawPo5=kd*gh0ws3tY&jB7y zDIFU-HFjdGq8RH=jh#9vwyM6qlatewDO2?EcBDyF z9ebKt#!ipLUW#XcXFx!JfkBz*u&~i#Vfe@=;u|Ps!2fK>0kg0EEEV#(miT@>Sg z{EJh*{wuo2MCHq$j`FYl6;5H2^5w6T@~{1=`)#@>fddU;9({|N4CXs~UrUtpDn9O6I?K!ZMn%QZ)X*cD7$)87p7h!lk1_ zjUaa7q{(qprcRqa*}HH5frEdfXj2azK63Qf@e?Ocoj!B+-1!S>e_p(l{@3LzSFc^a zar4&gI~jNH-OqgR@b9cgkDok!_WVWm%U7>+a`W=vynXln!^cmb6`fMLditfy7#Nl< zXH>pIMdM19t5mI4y++MiCbdoL)U9V$zk#_$!$yrQziZOe%DP!|n-<@sL|15#ts@hWau!@lEZ(B88?2y|MdC) zPapr^&i_vK4vtRFox61H*4@RmM^Cq2z5BTL?bqL9z(A4zC5Qj7$p3hs&KM*0z*q?{ z9u;yQE#%IU?oQNTd^;nyi$d!U7{g8b(qPj9?tw}d{3(i+ba#cj72FQc_JP~6P@I55 z`niST?~;0V#J>r*daO5at9F|D2{+l(O1f!e=Puo3PmpwzopYs|>`9jHZg8KJZVVNb z4Cy92jfRW#$p(#dcZS?yL3Cj9V6Y8S4q-M_2H;=yTY9* z-DI2I2$2q*m!))5o^+9JDwiPXrgNMw-IV{^q*rrgUryxoIBar-Unk7^8bBAqW+7lfpVP#0XLRl_125Py)Ug$NHObq7Xk4 zseAE#)%3j)Hwftk$|H&+dtrv4YVZpp9EXlX3@l&>_Kbk2dev`@SfSj;k&4ov#<-hQj)99Z#GUSsV$WW79dJt5Bk#p%kq#(V3)RK{}az7AJQv-p0br_%Q3)`=1+9w$(qb8-C$Zm}2krc}N@ zW(cgIt@wTs$REm;0L2XZ4d=%fDSRE5@--0W8i<^v8eyi4fkYw&nm77`U+)v8jQr+t zMb3RIegwClEFoLJU0Mpa^h>zN(ZaQ~=>9=CN1E{l!%{lma9CSBjR<8l&MbnLsDf~6 zWx=o|1hvHr`-s+}wt}MJRBOr7NNzL5^y4{IJe_abQ8cZhc2IeXmi=`rruzM*^;*Ni zFJ};f^9+aI4`-s<{I!niYA8yG&XZb*g0n5YpEY!2q&|X|rf7MQ$P<5vX!Vp5^(A9* zG|CONKZ!s)CAWGGRKw|fLts1AC912%e=4RH?N3J-;?RNuulrntRSdbBScLD=+l z8lse`*CZ>y9sXtiF(t$y8=^`GqnZ|u+z2fwHR^AQ+e-N$>ZKQI?bqR>kP2OO#v&G# zgUC~ACv?4_E2uTU{)F(b5NIilFYQY0&(!{lNNKnPb>{c?I>9{{?#>wD&=?^IZW?dW z_#&FS2jaS~C~<}2BouOQgPX>fG_Fa7yNlADj}g+jejvYN__UBa2W~ZegWxZ2Q@Gt= z0gYu^!A)b3?r=N8JrE;38jpCw-Cd5?q6_I03h6h)?T$Yhf6e#qrW#q6szsQw2PDl2uY zVU3MI6M)XJ+PeW(@mF`O&v_?3!3Q-5C$Dx)i zu66%(O)VTo7K$^YI;LKS>1tE-I$Suc@IGH(j~n4Uie8VSa7L8pRKF=t=nVbgR`2Z= z4Hvy4#ipL56Ql;G1L~53Iv{9dWMWjtxQ(8Xfwi8lo{_P!K^+4fgh$78#(h8B!vJ-` z>7X7s7t{xnz|vqcSO(k%8i0F2LogLA3!VhafoY%-coi%UW`GsIEU+S&4bsZMZ@@~R zGOn{y8885=fW}}|uqIdyGy|)HmS9cL2CN0P15H2;SR3pDnu2a%9nb@;3wna}zyQz; z3<89c&2B1sj2hU}G=|{0>Y8n}FNErr=)C3QPs9!INNfFb!-2 zUIpockr`k+FbixCW`pzs&l?aANEBs!XGH@TfK-OYAkCR-g4C~=fzHgnGWip0>p!*z(i0N zTnp-fnd_VteQ+E6rNM(>8So@%0H%Y6pypR+r5u<6zY+KpEDyc`D}Z_vP#-{J&=@oY zD}kvUlnWxRs%I)b zGz0g7^}(ZH127FV2XBHFU>4XA%mEvLN-XLFXas%-)&!e?7GP7*2DAcgL2J+*Yz}&Y z--AJ5D=->t4aR|Oz`0;sa5>lxOa|M7yTBj7RImeh7PJGef(*2FbVYw9Y|GB4?K%Oc3w8hxf_C6Z&;d*bM}kj59duM5Kt0f4GU_o{6*L6Rz_Oqf zXau$cO+ZJm12`BQ2?l^V=+L4;JunU|1I`5v!R26CFc~xgcY!8gD%b%$3+kZ5y9w%n zSzsA32Q;LPFb?~JMxYT`6Ep!Wzz(1-sDqBt1vCUbz_Oq(Xat6VCSVNM0ZagO&{3`h z4Z&?-S@0le0-gjrfEl2U8R`{i2z~&|f(BC%4ps$qEMNy{2-<*UL0bw3yHL0#?4fYb zm%_m?(p$kE(u31U4}gFZ zfjS!G7x_U8q9g1ic7dHl7uZR3LpufaV$e=O6EF;nYd(zuw!!ZN z(luWXJP1FX=}B-JNMitfFdcqmAs-9S*dPP`HQ-Zl9{2{NYm?se&dN^E7~BNXHHNNj zrtr@MX$(;Yw1S_mVKkO70NcU;3+M=*1Kq&Q;9&3+7y!-(BfIJ9NY`PA4p>h8hadt zKM16;NFy)}{s1D*r!06Aeix9&8qJ9CM}RaYD+lJlp8zU9cUIN`M&K5(Cg=)UfV)8( z@E}NI1{$l_!atq#@Y9&43;Z-@pfR2i=ng-PJ!ou0V<1oXBS8)P=z5eO__u=5AdNj} zOjsU_gTE(8V<}5;F8m|FX!z@a%i;F|Y3x`5Ooks#mX9GTg1g{f2&RI)Kn435183o< z`CJD4*5Fn6!$Af1w*WHXj|RKI{{xr}|0K`=@hgEeMq=Ox=x2aNGigqSaOl4WYr;PU zOhWvIpauN?U?Th$pbh-PL0fP(NMpjLU>Epnu8^o^;wu9e;2!9RG5&i)njYVxi zH~6Q3gTW1805}Sa1eb$duwN}O7XCrtbod*9@$mbCiC_wt3?2dZg4@88;9l@5cm~V@ zPk?UN-voRE|6q{%_WPjTtj@||&=mXwYz3|dUBD%v2llHC`og~y38bZYKFV;q zZ$%s3%MqlrC%=eC&k|^(88vMIGJb%(&qVipXrnnYZFC=0?`&G2jJ$2gXxdd$y1?MB|qC17O1PbjA6JyJX}SgZVJ+_~ zQN6In2&yyUP~EWRPYy+Wp!zY8?@M||eto8TL2f7Lscu+f&)#rTeXvH&>&o49kB6R= zih5xUtqXTYLEnYvevpr^VD;M~Tf+jXGgQyXP4$ZEI=Lx* zs_&F<#m|N6zgl+Q++Z2^DoweGwC>1JQ1wE5EOM9L}RF5f-L>UD0Jng~ri27zq zmBvG&T~U8b^@eN`c2IvyDbux@`dX@Il#im#i1?z-2z?0OSCoxtC-hbT*-ZQPK}{}N zheR6??Y3||bKw1k=u1R-i*_Q)+n(oF;j%0oUp;>(J~LAL8z){D#C3=6zN_cliQ6a6 zxf5@@;+*LX2KAgBcwH`h4uy{=>=Wmp9@~xQl6q_xUQ*)NeYhPWzg^_9?zBABJ!;(KUGZbylS7$j_;8T1Cr% zo=~a1shU<#)LgYbcav_a&;7W)5wiCXC8zE)K%R>z6MMct?c*SiFIsHTeMCzXHrw-Z zQ1gP?ifCuu`2HdeJ$WI!DBVuH97G)PzLR?HG@cf5L`@cPy7O_ih~p;PtFVu*O(IUG zLgmqy*FB-PFBIRckiM5JOG-!d8sfP0bV;>AJnIsA@s5yi)6=NJ<2gDr(cX!opC1dV zwt3;4RJS+!c-8I8Mb~MCC-h^a&2;S$y_8R({z+)We|qw$ULVDE*_)>+#yMgXEUqu& zS(#{a^qz$^di*c@im?#ousRl^Cu!<6!k?cnjg_s@bB}~xv~}8Fb&F?iLQnryJw4z4 z;-*oa7@Jw6HHZjk~R=7nSQr$Fq74?rwNOjX22CCZ!`T50NSpOwHJug$^(9``dZpwF& zzCX`X;SRvnP1OhU`vO8w&*D_Q7(oktm^^oCSM=oZOB||aB953r2zL~(E5c1rv?&ga zOZ=pp#vc@_j@#%h5^~cxNz91E_({wHL?0|>9>OikOLbG*6K?viy6IgO)g8?9M2tnm zam5%&)Loi$S>xE0&tj~m>giffby^(`RRAM5^MdAZ9@{H==x@x+2Ca)Z$flFfU^UfmOSLlnzr#Y6I z59C(Qo#r1SWj;_k6hb|Km|v@Pg`UrUanpQI+*J_sDN){Ze}VK=u42w2%0tWt>AfPF z<4~M198sO`h>>#Pa}XNwzwkMTIp8Q>U&Y)_99N7hsV!3Pqh9aHPc`_9pX#}Y8^Dh* z+`+tV3wH!>Yr-ALui;`|D2{{O_xAm`ime!J-*9{PXB{bR+&!gRth)x~Ze^Bd+Vptg z|5oa^k=~-_^F6HJ0?)|lHTW%PsP>XRasOXhmU!~}ke|Ht?cm3jO8TtlmyfWib;{3l zJ^MzcbCmR5e%g0}mHVkxrK{m}Wqke}U$-lB&#(r04UF{{RF!(Zj@X;1K^NGZ8%-{k zKUqrZ(#I3+2miPYaE zeZqLZ43=#D>zi7`YfC-JH+wK7yIqzhyO&43K=ig{tMFM_p*;z%SP)fIhWIsdcK}dvXb$oMZQoi{(veauYa*7SIM^?RX@<@$C#b)T@s;WaYr zH#U^<`TA_T=7m3DdGXKM{nXu5#^?H{VYi>K_6zq1eQIwZeRr|UAZ2ypF?8@{iuKl9be8{Ev zR&P!}V=ZRYY8Pr^CGFvQ$6FT9nW4`g&G(m8>q8#J*OB{ZDW#K$oO0zb1m&T zGrfGRZhM2u(*AskA7*C$g2i|3G;3y5M;V{%)5=A>VAG?OYr878lJVd0_}f!oFhAen z+D4~jJBsCct16|lnReRn1KZYjlJVbC`~Yo}Y&Pm;nf+;bYW{P*g@2E1*4t_D-U?~m zW&C#(|7evTvsvSzcMbmR>Ll&qddKtgv)Qs$jl5PK?jqyAr}!?bc4o7gjk<@|>E$fr zbG_A^OWEv}dGoc6v()zcf#P3X_bQtW@i^yG>a%Q*v0Se_F@DJm($cq#J6}(x|B>Rq z@oV;yz35uoxBo(UUa?%CUZ?v@_Upyl70%C9+w~`k?@`PDB}@3dRg(oeYJ9HO8#d`B zn>6U$iK#l>Wcr^ee$CwFFIkJJyB^lHk>?f5^>gp;e#y=S_?W*M(pkn=G>C82B<&@; z)+;Nz?BDWyHC!K-p7oOXjhxtfOS2v_AGm(=1LYMve<|hBwh)cbr*eI^cgA*n_W(?Sp3?lant8Xdt$l%>HI;j*zA^7oi6?) z&o`Cp-Cu;gVhMeQcU{v*teUI}~UNQSN-@R+GShfod*S~4G^cAzaK5=czA8n=m zTyJ2pp zzGjo!oO=}DE&D?a*DIf_U$gTgt!f;uB+o0B>pc!&oyYAqxtk{COFL7!es8^=ui3II zv0XhbsP$Q&(oeM>_L^NiR3p87Sy^5huFw8E-V~if6Wp*zC1SX zlk8VhxjtybtkQkv3tX}kP2TazEXHNTx~j~cFz+qdyG^EqSQZ$U+S znLk`_^Lp=V*7kaGv&RE!3VkZqyFEM(`|AvPVi49y=#?^*{?&x^*DRn_yzBEfvOZ|I zezS7#HFLTgb}UFy?dSUCOaVNxIrU7*P60j~V1J z>m{uEfn(B64cEK1tCqtOEQ3;GuE~6h<@%(s`Z+A>#DhOI_hq|F<$6t+bq?E8sbA%% zZ)HEJ7*hIc+qKVOs~=2Af4Qr*I4=#?+n#aEVfnkO9FM)C)=#bv_|7ed8T9saJl#{a zi&Ub+4AGED7F4@{RiVMIc(ISo>qb0>h;Bl(m%OwPY$d7sgcXM z`3^FFxPEWtLpf~8pW|xYfA1pnv0R_(aXN>&gszz$zekQwQiVSH63Q{X_JyP+>iDEQ zrJodWGl#Wt?&?^ZNxg>a+jV=8!{V20O7V|S)8~5MkI!;grD};uV~pkaFqP}GX65BD zyH$-2*tC~+DitXGgb$x`m|oe0_qNB>`pNZ{-OA)L${xqlgobXUnTLWfgI1oa(%m-19Mr~ zlan_5xT%A%Gl}arr+MYF9(D4XPZ=cZe=65oZVt$0)n{~EYZ@%`EraWALPK-eye`IV zTi0_H=_r-R{w^k?a+%wnuFw1p>(wsp{CCB5H)Q%6u8(apIhQ>%Zxwgo z`>rCsC)X#;osr8zYtQ*BGgP+gSfS6DlgqO6Z&?2tC(}>j`Ye-0xhy^4mxP8!^17MI z^?I7+xvXyQP0>RqIEwT$xZcfeO)i^S{kRhGvs%8D$^KR@$*_OUu=-_FRQtK!xYd?i zwmEb159YIFIcm7x#b8%1`_X^)4~-0@ot|9Z<Xj;F|G_bT<+4p5re%KmRjofR0xs{LGl_NN!Q%=+!*OHD)7{O5X$2|2ke zI$^<^%6f9Vk-_yoiFOO}Vw8wd{I*wJU zk^PZBndh<3>#l6ovr_Y)>uv5g&SNv27Mcy2DBGik>#gjp^4R>Rx2Ljd@_OXS^-1wI zdF;T}z^*@5RIjI8fAoB-JoYfa>-B-()b`8uC-vIrvA452&5b`N>v<~IXEy7Y$Ih

-(_yX1!z zSC7d4O~dt1eO&X{x>pUW^{QM!m6pU24lHiPS}wt42UP7PhhOuTL?j;qul z`%Qy=^H}!#=j&E}Qtjt@W3zxfb|~yzy-PW=ztV8M&8d-jY~=F_rt`+hcIV0U%L795 z*oxm1_6~0=>uoI8$9@XWV~NY&lp6Oz>XW!$nK>$tHJy@MK5&s*f4E*#VQe0I;{DRT zaYtF-GPvGz`nWtc(BM?&xlFbHtV#Aye;b>}T5QRlk+?wW&A9%iS6m+JeQ$iVSywVe zd1<&l^3b$A)~koR;l`)3zIk$ebj_K0>~jB>dOnZT>nYb8`OnT{51Lx8{CQ2DNGFNw z9e!aQ9=r1hS>KdeWWQC< zWqIt3S#HXvkFwm&xZW{pWgdI6=v83I5E);?^~)Em&SMP|W-mC@QuZ63T<^AVZ5|ss z`=E1!elj0oxjuH^hCDW9+_IK7Z5nxA z8m_mE*@yb@_+-8OIBBOR*Cz}(n8#wqEw0o-tImtL-pWRs$JX19GJ1bcUPqF+-al-?;1PJhsVq z+NFeka$IM|_3od~=CS?1t+n~iMIE>8nvdyutpD$~Q!Xj|%wugsH$P~;N#>hkO7?%~btjK)=IT@Jl4T9D9`Sr zOedD>is7BZ4KAk{$`lZ z?5oTSsT-x%AFhwTRxY1?+>!1%)vBq;w^*+C)v1urmhO5`PXD-!pTza)7RLE3NN4-y zT4rVmp?=d+tm+X%xCw9w(9=p8JWI@>ys9m#2v))5uM)bWqduWOLc+&rCHZ)mFapE7-meAY3s zSGiti)c(^qw{S0jERW!3+^2n8maL2pIeys>Cvf&qO>sY+QShi zOLXS!#r5=-2E9+x7jIZtK;H%Of4gO*;+@?M^8ME*I*L-SloC6r)Hjp+gw+cv;H)J1 zhq!c7euU7bhNl+_C;bVJqTy-9!jJSX8eV+=9RvS4yvmS&4mThA&*9yM{d4#MuYV4Y z_x|Vb-+cZ#{G)Ht@Z$NS7gaR;WU>646jL<(XtD4!u|>nL77I6-R5ZMJ{+mxO8eX{k zseDVviEwhKA#O?az+y_#bc!FR&D4Jm|NiHH4lf`7&*8U=g*z6@hr9EN?q9qddMqd! zUOXS_FDe>dyd2UKiiW2aJHGX{qT$8MY4Dvdc^gp3T|8c5#+P^-3b|v79Zxs&pN|(- z)t-v9pgfD`$JrYH9G+I|pTlp|{^#&64T^>ruP;ioe-2M>Q8c``{pKx;h8Hi-dL2bL z<-sl6FYc$(t=9A6`R~}KXu5^%-agn~ISQfeTl?=DVy~Qq6b>)EKj~)?^#}*AQ-d}}JDjHtcezJQ~T+#4d#q9T= zW?ztZk%ipF&%@;BqUjVrk45n!oQ^pX?r)XHz0J@=Xj8+Bmq*scqT$8QXLYggZN<)K z=oPVlrqo9k3qSX&X!^y^v-{hk;l=a&Kv@TFKmB`V=umJT#qF$&urAxn6vXv}d)Gu= zC1L9q-R2!|$LcDYLh;T+w;A!iwT1j%M(MGyj4}f8CqZUI)okmklB!xkj;?8kaWmn zh_VaEgP1_9Aa;-*kN`+DWHuxTvJH{~xeUpIeoGc(eK-?jdAoC%+AZH<2kkY$x z9*{PWZjgQuUr0D41~M122C@Tk6mlMN9r6(J24b|QjA9D0fjC13Lq<9DEb`I>Lfd}iC2ek09@uIr4xwW0 z<<3!ozKRkWp6cTP8#XOkeAkfR`hh?@8=gDfmNWLqB|+R6z&@88x`c&#Vf>@l2TTXj@E|m8vA*)b8N-%UXk3z$pv6gmJxStx1kJZ~x9Bs2+@5rFOA)}G)N`0i{6YA^d z78*z?DxVZAA704cOUEfXP+j0#8U7_#MJYRB0db|a(kYxT^xY{eFrrhBK3$bRbzP%^ zA_MKmMEbdh_7C*+a|rMXSI+3GPWRA)aIDYJ+t16F3Q585ad|xuX)E<~9HYX5aDtJt zPT_aWqGro{L5bWWi`@N!!`!?g1K__*eo-dwfhh9`N|;KytRbyTp2`Xjg?WmRzS1bgj{H2;F=2j+zK)|`kYD6iS)L;H@(PcP3Zn+oU%8{- zAH~JlNh}KE};2oG-8*;^XHK8pTVcEX9|$O5j0y}7 z4GBh->4Q2L;va#OR`7cLt)P&iEsU36Jxa$h9818v@4 z%39i|lY<);aG(@>I=Pa2Cr`+A_ar>m=j%8JMMVTqM%qUWAC9gR4_FEvn^O2d zJvPOV&C@AbQGV6$?-dw{o~jQDJIGJ0$*)-Px`3w2Z4&G8uP9|@y{3crS5E7?qJJefnf^|;sNU z)?r7*?Kb@Rkm^oKYHcUwWD_SPthtjC5aFaGk9NZP3r>pXG$*{Bg79@tisj#MKXX#{ zzC^q{C#B|RCuOdSvtl{GSuq&wtjzVHyMAA`9uGAKw%5|4u81uv5$}RS)J+GD>X1@W zXhcqKJxEQ6KBPRPG{g`>qdX&sfh6h3Ulu}P zAf3x(2!+Q%$ewwS?;tCse>F((RO6#pn=ifxom;V8v|`Ee-Df42YRz<(_VGFTYCwL_c1F|p2#9vFZ9|11;a=7IQg z5x*p9CJplP3B}0b%jOk<&psF0ydu4d_=`nXf1NQNK-61CE~|$JQ4Y_ z6-!Y52`ebLteWkOa+Mu>|m8vPKUu8-9lEVM1 z@zr!xs{8+k|9%b>{ff%hzEZ_DUAd`8ZmK%|Z5o%jiKNX+-2UxD|8Gj5)Tvy!S9(SVq?_2S_&7NX>z~fqZ}{>zx%n zhylb1QWatfv4Ge>Y$1*iH;6mL12P!m3GsylK!PA)kVr^0BnA=-nGQ*SEQe71wUEt_ zU66y2laMsXO-L5xDI^>60b;ztxnQhs3bzHs3epN<3vq_tP|P9q7t|`ODSq6l2#g%>tkeN_fhJ%QSQ|_P>w#%t70?%~4`zZD zKu3_~j%MBD+hsQ;8T$1lY>E;V@8A3KutrjHr{6V$>_Nt z%}KjJPjh)Ukk5Ugr_-DcHUK@KHwOoU7N95C5cCBbfkB|y)N%708nR-0A;-}Llh;@o zG1l0zpnPI!0YR_CNUS-; z6%qtd_vwJJfskm(V#x21bjWK+CB(CV^ne6IV#9<@*)PTT->r{3V*TH5ZjS%n{stGZ zE-Zx3MYW=^Th&p$9*cUKfP&$3e?6QBJ`qdzO{gwtAfM5<#)1=(0F^u4k1>N#Vex%H zbyJ^Dt9~V^l%~LaQvZwl9slhV57I0*D$u(fsaW8)$f>NlV;3b>SKVd@Q)&sfp7vDj z*{#UE`q|m@wGUtFp)t!|`FmK?J=J;tg5*vIr;O~|xc#jBu#6b{ zgSTeie!JrRlkppSIjnv->i*eR-!+e$*vQe+xW>jeTa51?JNnM?$%gdT`A_b48@up= zli%4B^D3<0l2q&dFV5xtY8XXLoEkOiR2Nqd`^*gs+IMKZcgM3nt{WZ<>3D2NhnXwq z5Bxl2efDazRQGl9ZN2{dICJ34s||g{>7 zVHD0+Id}HlS+TumKXxp8<(Dw}_l6Dh?Qo~(ti>Uf@mrT8m#qA1-Seoi=g*fct)h8c z=h~e&AKUPpY1Z!!4BD#yN871SSj&zF?Z-a&D?xktZ+7_O$Marsl}4Z4l*yXA47%%j zG$mo-!&7Yj(LNob9M4rvS-O?o_TQh--s!ix+b<+A3-jam4({*J!fWPO=GAw=q4C@M z#8f^oj9q!&d{*g{`BSRBcVrf9-J|yF8b6+p+=PwWygKusZ;W%=t-f|XKb1af?ml|* z=Xn!5&U^Pmt+7509a^Mrwyi(oPb@dSc(3yDN{5@%Jq&7Z&$>T#y>5r&aZdJSb}xx) zd!yM8Eq^q>n5#V^ZWPr0(Ml|(0i5RfIY@m4ZCJqw`tY0 z&zAEe8_pS^w>I_m;c@S?j@MoJ_w<4N=hfKs@_5X!gJCzV^I=r7mn~CM#f`9aNSo9y?(9?ST^e&|*TG=>?RBW?l(ZNAxqr{`? zj*VM1vuSMRN6$R*_@rawzV6P}?HZd!;4X!)R}h|pbZk7vFQPHCD__>uDVhw!2VNOj1lT zOpcu>bIff~8}{PKVO^?*FQH8_JCI(g3I?6~aU-(IiVXAQb*&W!1k@9q^hds@GV z!6!F+y{6orJ!^8WxW0cJ$L;%{pT+I_cWZL{{&8`5yW98oxZ#I?*o%MpV-$EAzlb5d z$y~C8yiL9$rr{X@-W`AMg?48BrxZ6M1Hs)xk=#9<_?C_113dxOI&*<#M@Cc2`o(Q%|V_>QnVs zW!uClbl!EwxTeSS4o&>wJ{A-Qe#XlvGMdcCx;`M^lAlNmBg$~GwlT)-#$sc&u>tEl zYsAz3bRZpx^-Z8t>D}~xtgx7t(l_WX`e)ie+p&0NvZ+|X=KkxS%uxmS)<)6`nE4N<+$o?#!czqNm{ z`#bZo)6Y6@Ie&4!aGJYUx>c_B(!IIfaW7U6&}n*t&e0F(HOPWb^w+wD-_y7KOn*%< zA9A46@DfFakx|B1*z<98A}wW4iFd?4aY}qI23SL_d#%Tf}00 zoORA_=L4s$OWc+221M;XZ?K-Irv=wv7Q*GgokkCq!|&%C_)*?oEHamxCFZAATiHRr zF5i{)c6aAh=U8yf75E}EUS^Od@hIO^`T#vf!`U!4l4Y`)Y(94JIkuVIhdMrr+#Djd zih!A54mNYlb>@fWr{*~`+PcO{vr4VMS^corWpajEuAWepszxQ+H`x>HHTE|9bGx(+tU@U?M`qXa+kZS-DlnP?yGLOyUqQRd&vFF{mzZ>x_Y;I8QyfS z*n7h};a&9F=-$YRX?nTdtXukR{Hy&Devbc)zdLx{RhMe33t?m^nM4*K%i9^k7-BRT zJ!lm@&i=+bi3i2=qFi)0`Qp-0++*(D-g)nU{>TqBUecyvWFmQ#JZg-g%V=-5f+v`ZtwJkU zYyIVoaL9B;kdSO5+0@XCN=) zTX;wDlK4iB0C8(?!U9BI1yFpd`RD!j)u-WV4agvCIzam@JM*vv2T zGoqP!+`P?NC&Sb(wO1Xs>+K{b%$=e~`A_)4#v>m8zZJQTq>?OSgX}V^TnQW$*L$QZxX1bYSW}1`CEbL~EnQP`@ zPxH-%W&tw3&|GO2nZ?Nb^=7HL$t*L=&8_ARGufJMJ&&uBYJwW%OmOaWW?^?%I7Ln! zaOkhjwQe8erh}Y(-|g&m^O!fud&Ilo{o*y#t@P!(y}n9!*4=aub?3){f!P}hK25Zn-dnAJ1?Xw z=xcNbJwd;wtywhV%xB}-9c&4ElD)|)SRFgV+VGjEj@A4*?pdkURBNvFy!Dy2TD~Z^ zfSq1d=hPK8wddMT*jFJsHsX`!)Hz(Z}`oI@&My-}7t0_vif3-FouFLY&_QdMb$wM!t_B4*;E3kd@>OvH<-5 zlko}Vi0NeZ2$Vn%Zt@rTW?scV;(NtN^RQ`KIo31QLCXXRoRX`cG>V-rZln97*GXTa zXX*{QT6gwu^KlQd!m!Y@6WWtf5^GEd9owNvbTGY*wnaqVVMp0lY%TwuJ0e;1Gmn{9 zS^vWtVU4%)tU7Cy93$_L&&$u`WOa{v2WspS^_`-q`hE5pyRFm7xeifz$oU%h-NNnV zdhSi`?LdNu+&%8Kp5+bl(!BZJQg5yInz!BC6N=U`?=!E9P6v-~*M{%-6a43c`&JTm zNjs2UgafM!$P=W5yiUF&V~u-^g{b%%<8!FYX3%jj(Jk}s$sNsFkdGA|?tPib^p^QJZzOcSV{hkFU{RCzWlPzRxdAV#a z$Eo-2kKF!xkN&If<@bfwd&=MCAM-#e}@u&1F`YpYI-N%+AsvUR&r(AN)lc7al;T}+Sfw{swVxBb5 zm_M5>EX&HYW?2iYJ=QVnlogPzu&U0o8#r^I9EA+MUCxsa%W}D2c2u6aP2H}hsK?bS zP=1x_kUF6*x37er?q&D42VzH7+ZA@TebPRQ{K|4JLIYk644Uh{=I!#1dY^hxdN%am z!@5N8LYAMY|EuZg#dh2OZ*$hN?RQ#Ga?0 z&|m3({;mG)ey;zR|Fpl&KY$}N3w;*mWpaj`Cru=U-bBmj7&Zawbp`vCMRSKQw&uvw zs!??WemZtPyVhx`FVk)H6{!EKbr&#IwBH2|0ry?MKX!i%&XD8J_4ED5{bFSLPXB%X zsDHu_-E}Og89rN$&(vUVt|YOf2k?LpNj!1`83I(eg^Ys|m_}w|-{z5p$e9)7X=s?| z$wu-jd6Vn}OIDI9@*(+zd`{}o{hTKkNtn^vXlq0pu|_xSa$kdkY5GHP4l`~xMjM$% zmT{*s%ec?THy*~`KLH*39D0sTsIYCuE@MA1{V+KDr11qZ;yYx1fVQA*Xa^cYyU?EW zI`jpKYV-y}=?HXF`4`Ed|c(XGhox z)&f{IoX_L)p+dLw-O!L<@F-Djo> zK$eopMzQgh5k|K{d9P(}fCtXANZx~cJd@}0r+EqA$Pe*Typ4EHY=kbX6km%Q&1L3` z<`(k=IJCVr%(?^hw^6z zF1ntcrx)pEOrld*&8pbvY!(>d1E|+uM1<)DM#B4GbS@WR?A4YYJx67Q7?i9Dk zUFVj$+o7O8aL0R-y?eX@?>(>Hdrrq9Hw*pH-7%Azg`T(5+u-Pcqxg8JkmjO|aK!+T zDjr27wulPx9+bve@i);Sgr+&r$8D|d)--E2_UlP&FDie8{6)5d9v!GQsLiTcT>!#H zLhsMBKe0PI-JP}01}NT6?qYW<5OamM4p{lFcgpLf$NEc9AI}E&Dz#bYnOd94Nz%M+pZ7uCuhjGY^4--4VPhu&n0eTRLQeYZUa z%>RJ>nEivD4J~vWYI%m6hpxo*9B&vpkO#d*9@Un1^(bAe*XftRxN-iqzCb@Z2?$(_ z-NijdGn<7xibxWywv|SS@q)3@c-7cyw4qVJmoz#Sj>L4DgZeo}PtjkYR`S?u>}@zD zt&o>CPk-kC}flYs~XzmL=dZ+=;&JpnOS{qdWXWt+6-QuRGhF7u~b& z&u({b9@wZv_w|SPGyOnLv(P+by}8j0kRdg!?v)=`~_al_n=D8^C^gU0IJeAhoB->nJ<_>AP26qlC5>X{s)nFwtc~gal5+x z+`HYw=xn}plf4n%{fJa4b~;xt1!|X}bKj>2`8WGx{b~MJ@IXR$R4lx-@6CaQCVYwY zSkEEjV}nu~zS4SVx#RQ;8UZ|cnC)YS*jX0FFXtWkV*Vsw!`q6^qL-K`9v7?8&3z)Jco>Jsd_0#-J+(d995uJLUnFZ zr_}f0jW9a~PETL^Mtc}^ax$EWADkt~fDOpsN+`guy&uqm zhkKHiGz;C`_F=M|JVn;RS2<1E0F51B?JBwn=yMWRa~B*j6ACmRe%NbJ^M4T^o7Y;0 z)qcCie#_mBuD7|D>pkohd987tPUtUuJrFqZCe+1;`co~Tp6>7;0e`LYUx$XdgeCXF z4O>hegO1rkcA<){hZ{D`xXrj1zRqc2dOO+`IBp};u4Fx+-iNa^HVs^V1Z+Q==kR=f z9R0^nyoKl>s2Bz{KTZrW-!=cyV&n%Nqfm^bdf5L~0G%-fpF76d8#AfjU zI-y@gbM!%-&2H!r1v>k|=B?-gA2MGvx0_?-ba@uNhOKT?)3N7s)I(~iTBlxy9yth1 zs8Jj3a{IXbm7RtjS-A=D(gs1zYp4a>bGNQp=$Um-A}9A_7f;aB^k*7i?N~?FlO-~W zD@bBDhGIGz%3(UYo6Thla1F13ZFb-q4x&@K8j5NN>iRwYF<%6?VMi#sRlw95IHJ9* zpih+w2PNIgfVP@sWreV9hgAWb?Jb2&mAA?anF$0~Bwv-E%6i!VKk(ciJ=AcOrqWf0 z%2f9`UEO6~fgif-YGbp|^T|38i`+#XhYtLeBtReJ0ht=0tTW-Nr+%xV6e&`OTf##uSd-Z^Fxd*wog^WPA zF$>E5eW3Rz^hbI<6y!*LJL+{Ee;wU!j7Wqlaa25R9)yy-#=6!L))TmzpJWTwO-;hS zRHF{Bum{*f>{0N%K19dyiyi0mLTCPnv&X4%PC2dJD0i59i+i`f5QtmqzwIA{whGG#ie1KIc^{sHbu5LBO@@+9gI8eV9D%szArChL3g&9;<1=t@ zcCdB4im11v!f}5A~cWBn{s9yQGWJ+sH9? z8*S-3bTWLPr=gqQLq`$`lo}`20mVDvtdpQ9+E`tzu~7J};imd>fSfOP%WITU1Jn>` zj1#p^=!fT!Z)z+22#&^QQ8zshzyTs2}TZ*|0uHe_! zW9k~5X(u|pLv|fFaDlVa8Q{L)Z-I~8Jmj-p$NDnHEasrMz5#VR1eH4qwHtH-Ch&$3 zXI%^5VlHrbg}DaKAUaF=mOL+8sk3S?dg*}sj{6Z=8@5Q?z~s-i@%*QI(BII~=D z1#ecsfvMC7p(3l%UDoP4eM;Bs27LkhjV}-R5q_i}20W~~{XYp*XK`uO!d8n5{z7if_ zF;9XMnj(h+gVJRBZzz-nX3degVApxz)va=etdiBT20U@lLQ|* z#Tg3!D$PlEGT^;WLQjwlCo9*kh(NILOcAOpmha;Eb4aLOie;?C- zCxaXHxqr&yKw--eU)7LWQb+!0HJ^e~{k@`_(4U6W2pUO)x}X7blqMFASVkkqPD4&lLO$mrmlq;dfs&S?p6Bmz;$I&*ibkXRjiqr=#|bo%63VHhE?l%^nu0z! z6)HX*bI(ll!dWyMT0WQlz89&#@$f(AasgCFFrQJIbuB}_&|q}NBl#^n5QXoj)miK!b|W{z}5v|wbQB%DN`mZMP3 zF{tM_RCEGrnxLvB>N*LPor2m Date: Thu, 6 May 2021 16:01:29 +0300 Subject: [PATCH 07/44] add contribs to fasttest --- docker/test/fasttest/run.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index a7cc398e5c9..da05998ae0b 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -185,6 +185,9 @@ function clone_submodules contrib/dragonbox contrib/fast_float contrib/NuRaft + contrib/lizard + contrib/density + contrib/lzsse ) git submodule sync From d60c70a8da0d9d07eaeff27a652de56e7d6164df Mon Sep 17 00:00:00 2001 From: fibersel <0583463@gmail.com> Date: Thu, 6 May 2021 16:17:55 +0300 Subject: [PATCH 08/44] add submodules --- contrib/density | 1 + contrib/lizard | 1 + contrib/lzsse | 1 + 3 files changed, 3 insertions(+) create mode 160000 contrib/density create mode 160000 contrib/lizard create mode 160000 contrib/lzsse diff --git a/contrib/density b/contrib/density new file mode 160000 index 00000000000..67bd584bd74 --- /dev/null +++ b/contrib/density @@ -0,0 +1 @@ +Subproject commit 67bd584bd7414d4bc1418e5b7fcf9d68e44c3f28 diff --git a/contrib/lizard b/contrib/lizard new file mode 160000 index 00000000000..af8518ccb8c --- /dev/null +++ b/contrib/lizard @@ -0,0 +1 @@ +Subproject commit af8518ccb8c68e062a8c80205ff07d56a2e77dd4 diff --git a/contrib/lzsse b/contrib/lzsse new file mode 160000 index 00000000000..1847c3e8279 --- /dev/null +++ b/contrib/lzsse @@ -0,0 +1 @@ +Subproject commit 1847c3e82794400deb56edd30d8aa3f445fd000b From 94b159c51039404a1577c8d16289c37b3c509239 Mon Sep 17 00:00:00 2001 From: fibersel <0583463@gmail.com> Date: Thu, 6 May 2021 16:34:27 +0300 Subject: [PATCH 09/44] add newlines --- contrib/density-cmake/CMakeLists.txt | 2 +- contrib/lizard-cmake/CMakeLists.txt | 2 +- contrib/lzsse-cmake/CMakeLists.txt | 2 +- src/Compression/CompressionCodecLZSSE2.h | 2 +- src/Compression/CompressionCodecLZSSE4.h | 2 +- src/Compression/CompressionCodecLZSSE8.h | 2 +- src/Compression/CompressionCodecLizard.h | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/contrib/density-cmake/CMakeLists.txt b/contrib/density-cmake/CMakeLists.txt index d48bbbf05b9..844459c4105 100644 --- a/contrib/density-cmake/CMakeLists.txt +++ b/contrib/density-cmake/CMakeLists.txt @@ -44,4 +44,4 @@ SET (Headers ADD_LIBRARY(density ${Sources} ${Headers}) -target_include_directories (density PUBLIC ${LIBRARY_DIR}) \ No newline at end of file +target_include_directories (density PUBLIC ${LIBRARY_DIR}) diff --git a/contrib/lizard-cmake/CMakeLists.txt b/contrib/lizard-cmake/CMakeLists.txt index eaf63db0acd..8aa40418ec1 100644 --- a/contrib/lizard-cmake/CMakeLists.txt +++ b/contrib/lizard-cmake/CMakeLists.txt @@ -16,4 +16,4 @@ SET (Headers ADD_LIBRARY(lizard ${Sources} ${Headers}) -target_include_directories (lizard PUBLIC ${LIBRARY_DIR}) \ No newline at end of file +target_include_directories (lizard PUBLIC ${LIBRARY_DIR}) diff --git a/contrib/lzsse-cmake/CMakeLists.txt b/contrib/lzsse-cmake/CMakeLists.txt index a463089a5b6..a4c0d9dd8e7 100644 --- a/contrib/lzsse-cmake/CMakeLists.txt +++ b/contrib/lzsse-cmake/CMakeLists.txt @@ -18,4 +18,4 @@ SET (Headers ADD_LIBRARY(lzsse ${Sources} ${Headers}) -target_include_directories (lzsse PUBLIC ${LIBRARY_DIR}) \ No newline at end of file +target_include_directories (lzsse PUBLIC ${LIBRARY_DIR}) diff --git a/src/Compression/CompressionCodecLZSSE2.h b/src/Compression/CompressionCodecLZSSE2.h index d26bb3e2cf2..b42ae4e7aca 100644 --- a/src/Compression/CompressionCodecLZSSE2.h +++ b/src/Compression/CompressionCodecLZSSE2.h @@ -30,4 +30,4 @@ private: const int level; }; -} \ No newline at end of file +} diff --git a/src/Compression/CompressionCodecLZSSE4.h b/src/Compression/CompressionCodecLZSSE4.h index ed1cc8fb26b..9c59e56a053 100644 --- a/src/Compression/CompressionCodecLZSSE4.h +++ b/src/Compression/CompressionCodecLZSSE4.h @@ -30,4 +30,4 @@ private: const int level; }; -} \ No newline at end of file +} diff --git a/src/Compression/CompressionCodecLZSSE8.h b/src/Compression/CompressionCodecLZSSE8.h index c9d006ca13b..3d51a5ead5b 100644 --- a/src/Compression/CompressionCodecLZSSE8.h +++ b/src/Compression/CompressionCodecLZSSE8.h @@ -30,4 +30,4 @@ private: const int level; }; -} \ No newline at end of file +} diff --git a/src/Compression/CompressionCodecLizard.h b/src/Compression/CompressionCodecLizard.h index 95bef7ab39e..8b39f224dbc 100644 --- a/src/Compression/CompressionCodecLizard.h +++ b/src/Compression/CompressionCodecLizard.h @@ -31,4 +31,4 @@ private: const int level; }; -} \ No newline at end of file +} From f01b53bafbd7e7450d51949b95291c40cdc808e2 Mon Sep 17 00:00:00 2001 From: fibersel <0583463@gmail.com> Date: Thu, 6 May 2021 16:47:23 +0300 Subject: [PATCH 10/44] add newline --- src/Compression/CompressionCodecDensity.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compression/CompressionCodecDensity.h b/src/Compression/CompressionCodecDensity.h index cfc8bfbd76a..ffdbd8f3619 100644 --- a/src/Compression/CompressionCodecDensity.h +++ b/src/Compression/CompressionCodecDensity.h @@ -32,4 +32,4 @@ private: const DENSITY_ALGORITHM algo; }; -} \ No newline at end of file +} From de69a9f3afbe5258ce79e80c6ddff0908320be32 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:04:25 +0300 Subject: [PATCH 11/44] Fix warning in LZSSE --- contrib/lzsse-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/lzsse-cmake/CMakeLists.txt b/contrib/lzsse-cmake/CMakeLists.txt index a4c0d9dd8e7..5ec5b312009 100644 --- a/contrib/lzsse-cmake/CMakeLists.txt +++ b/contrib/lzsse-cmake/CMakeLists.txt @@ -18,4 +18,4 @@ SET (Headers ADD_LIBRARY(lzsse ${Sources} ${Headers}) -target_include_directories (lzsse PUBLIC ${LIBRARY_DIR}) +target_include_directories (lzsse SYSTEM PUBLIC ${LIBRARY_DIR}) From 7afc052a11d7690e11ec772d017e504f972430d2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:12:50 +0300 Subject: [PATCH 12/44] Revert changes in compressor tool, because everything is already available in --codec argument --- programs/compressor/Compressor.cpp | 34 +----------------------------- 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index d5471a62afe..9f99f8eb95e 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -23,6 +23,7 @@ #include #include + namespace DB { namespace ErrorCodes @@ -80,11 +81,6 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) ("block-size,b", po::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") ("hc", "use LZ4HC instead of LZ4") ("zstd", "use ZSTD instead of LZ4") - ("lizard", "use Lizard instread of LZ4") - ("lzsse2", "use lzsse2 instread of LZ4") - ("lzsse4", "use lzsse4 instread of LZ4") - ("lzsse8", "use lzsse8 instread of LZ4") - ("density", "use Density instread of LZ4") ("param", po::value(), "extra params for compresion algorithm") ("codec", po::value>()->multitoken(), "use codecs combination instead of LZ4") ("level", po::value(), "compression level for codecs specified via flags") @@ -112,11 +108,6 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) bool decompress = options.count("decompress"); bool use_lz4hc = options.count("hc"); bool use_zstd = options.count("zstd"); - bool use_lizard = options.count("lizard"); - bool use_lzsse2 = options.count("lzsse2"); - bool use_lzsse4 = options.count("lzsse4"); - bool use_lzsse8 = options.count("lzsse8"); - bool use_density = options.count("density"); bool stat_mode = options.count("stat"); bool use_none = options.count("none"); unsigned block_size = options["block-size"].as(); @@ -124,10 +115,6 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) if (options.count("codec")) codecs = options["codec"].as>(); - std::optional param; - if (options.count("param")) - param = options["param"].as(); - if ((use_lz4hc || use_zstd || use_none) && !codecs.empty()) throw Exception("Wrong options, codec flags like --zstd and --codec options are mutually exclusive", ErrorCodes::BAD_ARGUMENTS); @@ -140,16 +127,6 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) method_family = "LZ4HC"; else if (use_zstd) method_family = "ZSTD"; - else if (use_lizard) - method_family = "Lizard"; - else if (use_lzsse2) - method_family = "LZSSE2"; - else if (use_lzsse4) - method_family = "LZSSE4"; - else if (use_lzsse8) - method_family = "LZSSE8"; - else if (use_density) - method_family = "Density"; else if (use_none) method_family = "NONE"; @@ -165,8 +142,6 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) std::string codecs_line = boost::algorithm::join(codecs, ","); auto ast = parseQuery(codec_parser, "(" + codecs_line + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); codec = CompressionCodecFactory::instance().get(ast, nullptr); - } else if (param.has_value()) { - codec = CompressionCodecFactory::instance().get(method_family, level, param); } else codec = CompressionCodecFactory::instance().get(method_family, level); @@ -185,9 +160,6 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) else wb = std::make_unique(STDOUT_FILENO); - - struct tms tv1, tv2; - times(&tv1); if (stat_mode) { /// Output statistic for compressed file. @@ -218,10 +190,6 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) CompressedWriteBuffer to(*wb, codec, block_size); copyData(*rb, to); } - times(&tv2); - - int tics_per_second = sysconf(_SC_CLK_TCK); - std::cerr << static_cast(tv2.tms_utime - tv1.tms_utime) / tics_per_second << std::endl; } catch (...) { From e330fe4bb5b68a7f8af4e4944c693afd619bcd53 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:15:11 +0300 Subject: [PATCH 13/44] Fix extra whitespace in exception messages --- src/Compression/CompressionCodecDensity.cpp | 4 ++-- src/Compression/CompressionCodecLZSSE2.cpp | 8 ++++---- src/Compression/CompressionCodecLZSSE4.cpp | 8 ++++---- src/Compression/CompressionCodecLZSSE8.cpp | 8 ++++---- src/Compression/CompressionCodecLizard.cpp | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Compression/CompressionCodecDensity.cpp b/src/Compression/CompressionCodecDensity.cpp index 841638ac6ed..8aec1f0098c 100644 --- a/src/Compression/CompressionCodecDensity.cpp +++ b/src/Compression/CompressionCodecDensity.cpp @@ -39,7 +39,7 @@ UInt32 CompressionCodecDensity::doCompressData(const char * source, UInt32 sourc { density_processing_result res = density_compress(reinterpret_cast(source), source_size, reinterpret_cast(dest), density_compress_safe_size(source_size), algo); if (res.state != DENSITY_STATE_OK) - throw Exception("Cannot compress block with Density; ", ErrorCodes::CANNOT_COMPRESS); + throw Exception("Cannot compress block with Density", ErrorCodes::CANNOT_COMPRESS); return res.bytesWritten; } @@ -47,7 +47,7 @@ void CompressionCodecDensity::doDecompressData(const char * source, UInt32 sourc { density_processing_result res = density_decompress(reinterpret_cast(source), source_size, reinterpret_cast(dest), density_decompress_safe_size(uncompressed_size)); if (res.state != DENSITY_STATE_OK) - throw Exception("Cannot decompress block with Density; ", ErrorCodes::CANNOT_DECOMPRESS); + throw Exception("Cannot decompress block with Density", ErrorCodes::CANNOT_DECOMPRESS); } void registerCodecDensity(CompressionCodecFactory & factory) diff --git a/src/Compression/CompressionCodecLZSSE2.cpp b/src/Compression/CompressionCodecLZSSE2.cpp index 215b93e6721..bbbe63690f0 100644 --- a/src/Compression/CompressionCodecLZSSE2.cpp +++ b/src/Compression/CompressionCodecLZSSE2.cpp @@ -40,12 +40,12 @@ UInt32 CompressionCodecLZSSE2::getMaxCompressedDataSize(UInt32 uncompressed_size UInt32 CompressionCodecLZSSE2::doCompressData(const char * source, UInt32 source_size, char * dest) const { UInt32 res; - LZSSE2_OptimalParseState* state = LZSSE2_MakeOptimalParseState(source_size); + LZSSE2_OptimalParseState * state = LZSSE2_MakeOptimalParseState(source_size); res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level); LZSSE2_FreeOptimalParseState(state); if (res == 0) - throw Exception("Cannot compress block with LZSSE2; ", ErrorCodes::CANNOT_COMPRESS); + throw Exception("Cannot compress block with LZSSE2", ErrorCodes::CANNOT_COMPRESS); return res; } @@ -54,9 +54,9 @@ void CompressionCodecLZSSE2::doDecompressData(const char * source, UInt32 source { UInt32 res; res = LZSSE2_Decompress(source, source_size, dest, uncompressed_size); - + if (res < uncompressed_size) - throw Exception("Cannot decompress block with LZSSE2; ", ErrorCodes::CANNOT_DECOMPRESS); + throw Exception("Cannot decompress block with LZSSE2", ErrorCodes::CANNOT_DECOMPRESS); } void registerCodecLZSSE2(CompressionCodecFactory & factory) diff --git a/src/Compression/CompressionCodecLZSSE4.cpp b/src/Compression/CompressionCodecLZSSE4.cpp index 9b687d866a1..1f5f00cdf91 100644 --- a/src/Compression/CompressionCodecLZSSE4.cpp +++ b/src/Compression/CompressionCodecLZSSE4.cpp @@ -40,12 +40,12 @@ UInt32 CompressionCodecLZSSE4::getMaxCompressedDataSize(UInt32 uncompressed_size UInt32 CompressionCodecLZSSE4::doCompressData(const char * source, UInt32 source_size, char * dest) const { UInt32 res; - LZSSE4_OptimalParseState* state = LZSSE4_MakeOptimalParseState(source_size); + LZSSE4_OptimalParseState * state = LZSSE4_MakeOptimalParseState(source_size); res = LZSSE4_CompressOptimalParse(state, source, source_size, dest, source_size, level); LZSSE4_FreeOptimalParseState(state); if (res == 0) - throw Exception("Cannot compress block with LZSSE4; ", ErrorCodes::CANNOT_COMPRESS); + throw Exception("Cannot compress block with LZSSE4", ErrorCodes::CANNOT_COMPRESS); return res; } @@ -54,9 +54,9 @@ void CompressionCodecLZSSE4::doDecompressData(const char * source, UInt32 source { UInt32 res; res = LZSSE4_Decompress(source, source_size, dest, uncompressed_size); - + if (res < uncompressed_size) - throw Exception("Cannot decompress block with LZSSE4; ", ErrorCodes::CANNOT_DECOMPRESS); + throw Exception("Cannot decompress block with LZSSE4", ErrorCodes::CANNOT_DECOMPRESS); } void registerCodecLZSSE4(CompressionCodecFactory & factory) diff --git a/src/Compression/CompressionCodecLZSSE8.cpp b/src/Compression/CompressionCodecLZSSE8.cpp index 04e73121fc6..e1e86c75f4e 100644 --- a/src/Compression/CompressionCodecLZSSE8.cpp +++ b/src/Compression/CompressionCodecLZSSE8.cpp @@ -39,12 +39,12 @@ UInt32 CompressionCodecLZSSE8::getMaxCompressedDataSize(UInt32 uncompressed_size UInt32 CompressionCodecLZSSE8::doCompressData(const char * source, UInt32 source_size, char * dest) const { UInt32 res; - LZSSE8_OptimalParseState* state = LZSSE8_MakeOptimalParseState(source_size); + LZSSE8_OptimalParseState * state = LZSSE8_MakeOptimalParseState(source_size); res = LZSSE8_CompressOptimalParse(state, source, source_size, dest, source_size, level); LZSSE8_FreeOptimalParseState(state); if (res == 0) - throw Exception("Cannot compress block with LZSSE; ", ErrorCodes::CANNOT_COMPRESS); + throw Exception("Cannot compress block with LZSSE", ErrorCodes::CANNOT_COMPRESS); return res; } @@ -53,9 +53,9 @@ void CompressionCodecLZSSE8::doDecompressData(const char * source, UInt32 source { UInt32 res; res = LZSSE8_Decompress(source, source_size, dest, uncompressed_size); - + if (res < uncompressed_size) - throw Exception("Cannot decompress block with LZSSE; ", ErrorCodes::CANNOT_DECOMPRESS); + throw Exception("Cannot decompress block with LZSSE", ErrorCodes::CANNOT_DECOMPRESS); } void registerCodecLZSSE8(CompressionCodecFactory & factory) diff --git a/src/Compression/CompressionCodecLizard.cpp b/src/Compression/CompressionCodecLizard.cpp index a4106965bd5..2006a708493 100644 --- a/src/Compression/CompressionCodecLizard.cpp +++ b/src/Compression/CompressionCodecLizard.cpp @@ -42,7 +42,7 @@ UInt32 CompressionCodecLizard::doCompressData(const char * source, UInt32 source int res = Lizard_compress(source, dest, source_size, Lizard_compressBound(source_size), level); if (res == 0) - throw Exception("Cannot compress block with Lizard; ", ErrorCodes::CANNOT_COMPRESS); + throw Exception("Cannot compress block with Lizard", ErrorCodes::CANNOT_COMPRESS); return res; } @@ -51,7 +51,7 @@ void CompressionCodecLizard::doDecompressData(const char * source, UInt32 source int res = Lizard_decompress_safe(source, dest, source_size, uncompressed_size); if (res < 0) - throw Exception("Cannot compress block with Lizard; ", ErrorCodes::CANNOT_DECOMPRESS); + throw Exception("Cannot compress block with Lizard", ErrorCodes::CANNOT_DECOMPRESS); } void registerCodecLizard(CompressionCodecFactory & factory) From 0da00f6497bb91b014c2811ced5a83743eef83c6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:18:24 +0300 Subject: [PATCH 14/44] Fix style, fix typo and remove extra code --- src/Compression/CompressionCodecDensity.cpp | 42 +++++++-------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/src/Compression/CompressionCodecDensity.cpp b/src/Compression/CompressionCodecDensity.cpp index 8aec1f0098c..5a738c355c1 100644 --- a/src/Compression/CompressionCodecDensity.cpp +++ b/src/Compression/CompressionCodecDensity.cpp @@ -59,44 +59,30 @@ void registerCodecDensity(CompressionCodecFactory & factory) [&](const ASTPtr & arguments) -> CompressionCodecPtr { DENSITY_ALGORITHM algo = CompressionCodecDensity::DENSITY_DEFAULT_ALGO; - //std::cerr << arguments << std::endl; + if (arguments && !arguments->children.empty()) { if (arguments->children.size() != 1) throw Exception( - "Deisnty codec must have 1 parameter, given " + std::to_string(arguments->children.size()), + "Density codec must have only one parameter, given " + std::to_string(arguments->children.size()), ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE); const auto children = arguments->children; const auto * algo_literal = children[0]->as(); - if (!algo_literal) - throw Exception("Density codec argument must be string (algorithm)", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + if (!algo_literal || algo_literal->value.getType() != Field::Types::String) + throw Exception("Density codec argument must be string (algorithm), one of: 'lion', 'chameleon', 'cheetah'", + ErrorCodes::ILLEGAL_CODEC_PARAMETER); - - if (algo_literal->value.getType() == Field::Types::Which::UInt64) { - const auto algorithm = algo_literal->value.safeGet(); - if (algorithm == 3) { - algo = DENSITY_ALGORITHM_LION; - } else if (algorithm == 2) { - algo = DENSITY_ALGORITHM_CHEETAH; - } else if (algorithm == 1) { - algo = DENSITY_ALGORITHM_CHAMELEON; - } else { - throw Exception("Density codec argument may be LION, CHAMELEON, CHEETAH", ErrorCodes::ILLEGAL_CODEC_PARAMETER); - } - } else { - const auto algorithm = algo_literal->value.safeGet(); - if (algorithm == "LION") { - algo = DENSITY_ALGORITHM_LION; - } else if (algorithm == "CHAMELEON") { - algo = DENSITY_ALGORITHM_CHAMELEON; - } else if (algorithm == "CHEETAH") { - algo = DENSITY_ALGORITHM_CHEETAH; - } else { - throw Exception("Density codec argument may be LION, CHAMELEON, CHEETAH", ErrorCodes::ILLEGAL_CODEC_PARAMETER); - } - } + const auto algorithm = algo_literal->value.safeGet(); + if (algorithm == "lion") + algo = DENSITY_ALGORITHM_LION; + else if (algorithm == "chameleon") + algo = DENSITY_ALGORITHM_CHAMELEON; + else if (algorithm == "cheetah") + algo = DENSITY_ALGORITHM_CHEETAH; + else + throw Exception("Density codec argument may be one of: 'lion', 'chameleon', 'cheetah'", ErrorCodes::ILLEGAL_CODEC_PARAMETER); } return std::make_shared(algo); From fff2f7b2be4f42c9212a1735364bac0e6808c11e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:21:21 +0300 Subject: [PATCH 15/44] Fix style, fix typo and remove extra code --- tests/integration/test_non_default_compression/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_non_default_compression/test.py b/tests/integration/test_non_default_compression/test.py index 678b2167595..9689b4110a2 100644 --- a/tests/integration/test_non_default_compression/test.py +++ b/tests/integration/test_non_default_compression/test.py @@ -143,7 +143,7 @@ def test_experimental_codecs(start_cluster): node6.query(""" CREATE TABLE compression_experimental_codecs ( somedate Date CODEC(Lizard(12)), - id UInt64 CODEC(Density('LION')), + id UInt64 CODEC(Density('lion')), data String CODEC(LZSSE4(3)) ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; """) From 4cfabd2675666039cd1c2b45cafc3eabd21c2107 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:21:48 +0300 Subject: [PATCH 16/44] Fix style, fix typo and remove extra code --- src/Compression/CompressionCodecDensity.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Compression/CompressionCodecDensity.cpp b/src/Compression/CompressionCodecDensity.cpp index 5a738c355c1..4404b890899 100644 --- a/src/Compression/CompressionCodecDensity.cpp +++ b/src/Compression/CompressionCodecDensity.cpp @@ -63,9 +63,8 @@ void registerCodecDensity(CompressionCodecFactory & factory) if (arguments && !arguments->children.empty()) { if (arguments->children.size() != 1) - throw Exception( - "Density codec must have only one parameter, given " + std::to_string(arguments->children.size()), - ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE); + throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, + "Density codec must have only one parameter, given {}", arguments->children.size()); const auto children = arguments->children; From 1431679f516010160ae45fbd0a16c1119c95eba6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:23:11 +0300 Subject: [PATCH 17/44] Remove Lizard framing format because we do not need it --- contrib/lizard-cmake/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/contrib/lizard-cmake/CMakeLists.txt b/contrib/lizard-cmake/CMakeLists.txt index 8aa40418ec1..ad5f5bf132d 100644 --- a/contrib/lizard-cmake/CMakeLists.txt +++ b/contrib/lizard-cmake/CMakeLists.txt @@ -4,14 +4,11 @@ SET (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/lizard") SET (Sources "${LIBRARY_DIR}/lib/lizard_decompress.c" "${LIBRARY_DIR}/lib/lizard_compress.c" - "${LIBRARY_DIR}/lib/lizard_frame.c" - "${LIBRARY_DIR}/lib/xxhash/xxhash.c" ) SET (Headers "${LIBRARY_DIR}/lib/lizard_compress.h" "${LIBRARY_DIR}/lib/lizard_common.h" - "${LIBRARY_DIR}/lib/lizard_frame.h" ) ADD_LIBRARY(lizard ${Sources} ${Headers}) From e000b994e5e2431c177e4dc858944ed9ef9ba795 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:26:31 +0300 Subject: [PATCH 18/44] Add missing whitespace --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 41d5705187f..d005966ed6d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -238,7 +238,7 @@ class IColumn; M(Bool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \ M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \ - M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs(LZSSE*, Lizard, Density).", 0) \ + M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (LZSSE2/4/8, Lizard, Density). These codecs are provided for evaluation purposes.", 0) \ M(UInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \ M(UInt64, query_profiler_real_time_period_ns, 1000000000, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(UInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ From ecc2e98265351001c5f66049a4626d3be4c3def9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:30:17 +0300 Subject: [PATCH 19/44] Fix wrong intent in exception message --- src/Compression/CompressionFactory.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index 9ca64e36468..6e7444ca508 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -78,7 +78,8 @@ void CompressionCodecFactory::validateCodec(const String & family_name, std::opt } } -ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const IDataType * column_type, bool sanity_check, bool allow_experimental_codecs) const +ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( + const ASTPtr & ast, const IDataType * column_type, bool sanity_check, bool allow_experimental_codecs) const { if (const auto * func = ast->as()) { @@ -107,16 +108,21 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr else throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - if (sanity_check && !allow_experimental_codecs) { - if (codec_family_name == "Lizard" || + if (!allow_experimental_codecs) + { + if (codec_family_name == "Lizard" || codec_family_name == "Density" || codec_family_name == "LZSSE2" || codec_family_name == "LZSSE4" || - codec_family_name == "LZSSE8") { + codec_family_name == "LZSSE8") + { throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Experimental codecs Lizard, Density and LZSSE* are not allowed, please enable allow_experimental_codecs flag."); + "Codec {} is experimental and not meant to be used in production." + " You can enable it with the 'allow_experimental_codecs' setting.", + codec_family_name); } } + /// Default codec replaced with current default codec which may depend on different /// settings (and properties of data) in runtime. CompressionCodecPtr result_codec; From b0df9286f0c88e6939f5a9119ca4f63c8120a36f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:30:53 +0300 Subject: [PATCH 20/44] Fix whitespace --- src/Compression/CompressionFactory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index 6e7444ca508..a703040e98c 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -116,7 +116,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( codec_family_name == "LZSSE4" || codec_family_name == "LZSSE8") { - throw Exception(ErrorCodes::BAD_ARGUMENTS, + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec {} is experimental and not meant to be used in production." " You can enable it with the 'allow_experimental_codecs' setting.", codec_family_name); From a25edc192ca70743ec535f462251ecbb51b3840f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:31:58 +0300 Subject: [PATCH 21/44] Fix whitespace --- src/Compression/CompressionCodecLZSSE4.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Compression/CompressionCodecLZSSE4.cpp b/src/Compression/CompressionCodecLZSSE4.cpp index 1f5f00cdf91..7189a9cee55 100644 --- a/src/Compression/CompressionCodecLZSSE4.cpp +++ b/src/Compression/CompressionCodecLZSSE4.cpp @@ -18,7 +18,6 @@ namespace ErrorCodes CompressionCodecLZSSE4::CompressionCodecLZSSE4(int level_) : level(level_) { - setCodecDescription("LZSSE4", {std::make_shared(static_cast(level))}); } From 1a091c7a17e5fc165553a39f2d6497232144d9d0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:32:22 +0300 Subject: [PATCH 22/44] Fix unbalanced whitespace --- src/Compression/CompressionCodecLZSSE2.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compression/CompressionCodecLZSSE2.h b/src/Compression/CompressionCodecLZSSE2.h index b42ae4e7aca..d644645e3ac 100644 --- a/src/Compression/CompressionCodecLZSSE2.h +++ b/src/Compression/CompressionCodecLZSSE2.h @@ -5,10 +5,10 @@ namespace DB { + class CompressionCodecLZSSE2 : public ICompressionCodec { public: - CompressionCodecLZSSE2(int level_); uint8_t getMethodByte() const override; From 77414b50e6a2e065d6bd7ccd08f67fdf3ad5fcdf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 03:55:16 +0300 Subject: [PATCH 23/44] Remove copy-paste --- src/Compression/CompressionCodecDensity.cpp | 1 - src/Compression/CompressionCodecLZSSE.cpp | 129 ++++++++++++++++++ ...nCodecLZSSE8.h => CompressionCodecLZSSE.h} | 14 +- src/Compression/CompressionCodecLZSSE2.cpp | 91 ------------ src/Compression/CompressionCodecLZSSE2.h | 33 ----- src/Compression/CompressionCodecLZSSE4.cpp | 90 ------------ src/Compression/CompressionCodecLZSSE4.h | 33 ----- src/Compression/CompressionCodecLZSSE8.cpp | 90 ------------ src/Compression/CompressionCodecLizard.cpp | 1 - src/Compression/CompressionFactory.cpp | 27 ++-- src/Compression/CompressionInfo.h | 2 +- src/Storages/System/StorageSystemErrors.cpp | 3 +- 12 files changed, 152 insertions(+), 362 deletions(-) create mode 100644 src/Compression/CompressionCodecLZSSE.cpp rename src/Compression/{CompressionCodecLZSSE8.h => CompressionCodecLZSSE.h} (79%) delete mode 100644 src/Compression/CompressionCodecLZSSE2.cpp delete mode 100644 src/Compression/CompressionCodecLZSSE2.h delete mode 100644 src/Compression/CompressionCodecLZSSE4.cpp delete mode 100644 src/Compression/CompressionCodecLZSSE4.h delete mode 100644 src/Compression/CompressionCodecLZSSE8.cpp diff --git a/src/Compression/CompressionCodecDensity.cpp b/src/Compression/CompressionCodecDensity.cpp index 4404b890899..bd1fb25e18f 100644 --- a/src/Compression/CompressionCodecDensity.cpp +++ b/src/Compression/CompressionCodecDensity.cpp @@ -2,7 +2,6 @@ #include #include #include -#include namespace DB diff --git a/src/Compression/CompressionCodecLZSSE.cpp b/src/Compression/CompressionCodecLZSSE.cpp new file mode 100644 index 00000000000..05c5f507e24 --- /dev/null +++ b/src/Compression/CompressionCodecLZSSE.cpp @@ -0,0 +1,129 @@ +#include +#include +#include +#include + +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CANNOT_COMPRESS; + extern const int CANNOT_DECOMPRESS; + extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; + extern const int ILLEGAL_CODEC_PARAMETER; + extern const int LOGICAL_ERROR; +} + +CompressionCodecLZSSE::CompressionCodecLZSSE(UInt32 type_, UInt32 level_) : type(type_), level(level_) +{ + if (type != 2 && type != 4 && type != 8) + throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no LZSSE{} codec", type); + + setCodecDescription(fmt::format("LZSSE{}", type), {std::make_shared(static_cast(level))}); +} + +uint8_t CompressionCodecLZSSE::getMethodByte() const +{ + switch (type) + { + case 2: return static_cast(CompressionMethodByte::LZSSE2); + case 4: return static_cast(CompressionMethodByte::LZSSE4); + case 8: return static_cast(CompressionMethodByte::LZSSE8); + default: + throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no LZSSE{} codec", type); + } +} + +void CompressionCodecLZSSE::updateHash(SipHash & hash) const +{ + getCodecDesc()->updateTreeHash(hash); +} + +UInt32 CompressionCodecLZSSE::getMaxCompressedDataSize(UInt32 uncompressed_size) const +{ + return uncompressed_size; +} + +UInt32 CompressionCodecLZSSE::doCompressData(const char * source, UInt32 source_size, char * dest) const +{ + UInt32 res = 0; + switch (type) + { + case 2: + { + LZSSE2_OptimalParseState * state = LZSSE2_MakeOptimalParseState(source_size); + res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level); + LZSSE2_FreeOptimalParseState(state); + break; + } + case 4: + { + LZSSE2_OptimalParseState * state = LZSSE2_MakeOptimalParseState(source_size); + res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level); + LZSSE2_FreeOptimalParseState(state); + break; + } + case 8: + { + LZSSE2_OptimalParseState * state = LZSSE2_MakeOptimalParseState(source_size); + res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level); + LZSSE2_FreeOptimalParseState(state); + break; + } + default: + break; + } + + if (res == 0) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress block with LZSSE{}", type); + return res; +} + +void CompressionCodecLZSSE::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const +{ + UInt32 res = LZSSE2_Decompress(source, source_size, dest, uncompressed_size); + if (res < uncompressed_size) + throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress block with LZSSE{}", type); +} + +void registerCodecsLZSSE(CompressionCodecFactory & factory) +{ + for (auto [type, method_byte] : std::initializer_list> + { + {2, CompressionMethodByte::LZSSE2}, + {4, CompressionMethodByte::LZSSE4}, + {8, CompressionMethodByte::LZSSE8} + }) + { + factory.registerCompressionCodec( + fmt::format("LZSSE{}", type), + uint8_t(method_byte), + [type = type](const ASTPtr & arguments) -> CompressionCodecPtr + { + int level = 1; + if (arguments && !arguments->children.empty()) + { + if (arguments->children.size() != 1) + throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, + "LZSSE{} codec must have 1 parameter, {} given", type, arguments->children.size()); + + const auto children = arguments->children; + const auto * level_literal = children[0]->as(); + if (!level_literal) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, + "LZSSE{} first codec argument must be integer", type); + + level = level_literal->value.safeGet(); + } + + return std::make_shared(type, level); + }); + } +} + +} diff --git a/src/Compression/CompressionCodecLZSSE8.h b/src/Compression/CompressionCodecLZSSE.h similarity index 79% rename from src/Compression/CompressionCodecLZSSE8.h rename to src/Compression/CompressionCodecLZSSE.h index 3d51a5ead5b..85cc59c7be0 100644 --- a/src/Compression/CompressionCodecLZSSE8.h +++ b/src/Compression/CompressionCodecLZSSE.h @@ -5,29 +5,25 @@ namespace DB { -class CompressionCodecLZSSE8 : public ICompressionCodec + +class CompressionCodecLZSSE : public ICompressionCodec { public: - - CompressionCodecLZSSE8(int level_); + CompressionCodecLZSSE(UInt32 type_, UInt32 level_); uint8_t getMethodByte() const override; - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - void updateHash(SipHash & hash) const override; protected: UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - bool isCompression() const override { return true; } - bool isGenericCompression() const override { return true; } private: - const int level; + const UInt32 type; + const UInt32 level; }; } diff --git a/src/Compression/CompressionCodecLZSSE2.cpp b/src/Compression/CompressionCodecLZSSE2.cpp deleted file mode 100644 index bbbe63690f0..00000000000 --- a/src/Compression/CompressionCodecLZSSE2.cpp +++ /dev/null @@ -1,91 +0,0 @@ -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int CANNOT_COMPRESS; - extern const int CANNOT_DECOMPRESS; - extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; - extern const int ILLEGAL_CODEC_PARAMETER; -} - -CompressionCodecLZSSE2::CompressionCodecLZSSE2(int level_) : level(level_) -{ - - setCodecDescription("LZSSE2", {std::make_shared(static_cast(level))}); -} - -uint8_t CompressionCodecLZSSE2::getMethodByte() const -{ - return static_cast(CompressionMethodByte::LZSSE2); -} - -void CompressionCodecLZSSE2::updateHash(SipHash & hash) const -{ - getCodecDesc()->updateTreeHash(hash); -} - -UInt32 CompressionCodecLZSSE2::getMaxCompressedDataSize(UInt32 uncompressed_size) const -{ - return uncompressed_size; -} - -UInt32 CompressionCodecLZSSE2::doCompressData(const char * source, UInt32 source_size, char * dest) const -{ - UInt32 res; - LZSSE2_OptimalParseState * state = LZSSE2_MakeOptimalParseState(source_size); - res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level); - LZSSE2_FreeOptimalParseState(state); - - if (res == 0) - throw Exception("Cannot compress block with LZSSE2", ErrorCodes::CANNOT_COMPRESS); - - return res; -} - -void CompressionCodecLZSSE2::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const -{ - UInt32 res; - res = LZSSE2_Decompress(source, source_size, dest, uncompressed_size); - - if (res < uncompressed_size) - throw Exception("Cannot decompress block with LZSSE2", ErrorCodes::CANNOT_DECOMPRESS); -} - -void registerCodecLZSSE2(CompressionCodecFactory & factory) -{ - UInt8 method_code = UInt8(CompressionMethodByte::LZSSE2); - factory.registerCompressionCodec( - "LZSSE2", - method_code, - [&](const ASTPtr & arguments) -> CompressionCodecPtr - { - int level = 1; - if (arguments && !arguments->children.empty()) - { - if (arguments->children.size() != 1) - throw Exception( - "LZSSE2 codec must have 1 parameter, given " + std::to_string(arguments->children.size()), - ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE); - - const auto children = arguments->children; - const auto * level_literal = children[0]->as(); - if (!level_literal) - throw Exception("LZSSE2 first codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); - - level = level_literal->value.safeGet(); - - } - - return std::make_shared(level); - }); -} - -} diff --git a/src/Compression/CompressionCodecLZSSE2.h b/src/Compression/CompressionCodecLZSSE2.h deleted file mode 100644 index d644645e3ac..00000000000 --- a/src/Compression/CompressionCodecLZSSE2.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ - -class CompressionCodecLZSSE2 : public ICompressionCodec -{ -public: - CompressionCodecLZSSE2(int level_); - - uint8_t getMethodByte() const override; - - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - - void updateHash(SipHash & hash) const override; - -protected: - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - - bool isCompression() const override { return true; } - - bool isGenericCompression() const override { return true; } - -private: - const int level; -}; - -} diff --git a/src/Compression/CompressionCodecLZSSE4.cpp b/src/Compression/CompressionCodecLZSSE4.cpp deleted file mode 100644 index 7189a9cee55..00000000000 --- a/src/Compression/CompressionCodecLZSSE4.cpp +++ /dev/null @@ -1,90 +0,0 @@ -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int CANNOT_COMPRESS; - extern const int CANNOT_DECOMPRESS; - extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; - extern const int ILLEGAL_CODEC_PARAMETER; -} - -CompressionCodecLZSSE4::CompressionCodecLZSSE4(int level_) : level(level_) -{ - setCodecDescription("LZSSE4", {std::make_shared(static_cast(level))}); -} - -uint8_t CompressionCodecLZSSE4::getMethodByte() const -{ - return static_cast(CompressionMethodByte::LZSSE4); -} - -void CompressionCodecLZSSE4::updateHash(SipHash & hash) const -{ - getCodecDesc()->updateTreeHash(hash); -} - -UInt32 CompressionCodecLZSSE4::getMaxCompressedDataSize(UInt32 uncompressed_size) const -{ - return uncompressed_size; -} - -UInt32 CompressionCodecLZSSE4::doCompressData(const char * source, UInt32 source_size, char * dest) const -{ - UInt32 res; - LZSSE4_OptimalParseState * state = LZSSE4_MakeOptimalParseState(source_size); - res = LZSSE4_CompressOptimalParse(state, source, source_size, dest, source_size, level); - LZSSE4_FreeOptimalParseState(state); - - if (res == 0) - throw Exception("Cannot compress block with LZSSE4", ErrorCodes::CANNOT_COMPRESS); - - return res; -} - -void CompressionCodecLZSSE4::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const -{ - UInt32 res; - res = LZSSE4_Decompress(source, source_size, dest, uncompressed_size); - - if (res < uncompressed_size) - throw Exception("Cannot decompress block with LZSSE4", ErrorCodes::CANNOT_DECOMPRESS); -} - -void registerCodecLZSSE4(CompressionCodecFactory & factory) -{ - UInt8 method_code = UInt8(CompressionMethodByte::LZSSE4); - factory.registerCompressionCodec( - "LZSSE4", - method_code, - [&](const ASTPtr & arguments) -> CompressionCodecPtr - { - int level = 1; - if (arguments && !arguments->children.empty()) - { - if (arguments->children.size() != 1) - throw Exception( - "LZSSE4 codec must have 1 parameter, given " + std::to_string(arguments->children.size()), - ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE); - - const auto children = arguments->children; - const auto * level_literal = children[0]->as(); - if (!level_literal) - throw Exception("LZSSE4 first codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); - - level = level_literal->value.safeGet(); - - } - - return std::make_shared(level); - }); -} - -} diff --git a/src/Compression/CompressionCodecLZSSE4.h b/src/Compression/CompressionCodecLZSSE4.h deleted file mode 100644 index 9c59e56a053..00000000000 --- a/src/Compression/CompressionCodecLZSSE4.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ -class CompressionCodecLZSSE4 : public ICompressionCodec -{ -public: - - CompressionCodecLZSSE4(int level_); - - uint8_t getMethodByte() const override; - - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - - void updateHash(SipHash & hash) const override; - -protected: - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - - bool isCompression() const override { return true; } - - bool isGenericCompression() const override { return true; } - -private: - const int level; -}; - -} diff --git a/src/Compression/CompressionCodecLZSSE8.cpp b/src/Compression/CompressionCodecLZSSE8.cpp deleted file mode 100644 index e1e86c75f4e..00000000000 --- a/src/Compression/CompressionCodecLZSSE8.cpp +++ /dev/null @@ -1,90 +0,0 @@ -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int CANNOT_COMPRESS; - extern const int CANNOT_DECOMPRESS; - extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; - extern const int ILLEGAL_CODEC_PARAMETER; -} - -CompressionCodecLZSSE8::CompressionCodecLZSSE8(int level_) : level(level_) -{ - setCodecDescription("LZSSE8", {std::make_shared(static_cast(level))}); -} - -uint8_t CompressionCodecLZSSE8::getMethodByte() const -{ - return static_cast(CompressionMethodByte::LZSSE8); -} - -void CompressionCodecLZSSE8::updateHash(SipHash & hash) const -{ - getCodecDesc()->updateTreeHash(hash); -} - -UInt32 CompressionCodecLZSSE8::getMaxCompressedDataSize(UInt32 uncompressed_size) const -{ - return uncompressed_size; -} - -UInt32 CompressionCodecLZSSE8::doCompressData(const char * source, UInt32 source_size, char * dest) const -{ - UInt32 res; - LZSSE8_OptimalParseState * state = LZSSE8_MakeOptimalParseState(source_size); - res = LZSSE8_CompressOptimalParse(state, source, source_size, dest, source_size, level); - LZSSE8_FreeOptimalParseState(state); - - if (res == 0) - throw Exception("Cannot compress block with LZSSE", ErrorCodes::CANNOT_COMPRESS); - - return res; -} - -void CompressionCodecLZSSE8::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const -{ - UInt32 res; - res = LZSSE8_Decompress(source, source_size, dest, uncompressed_size); - - if (res < uncompressed_size) - throw Exception("Cannot decompress block with LZSSE", ErrorCodes::CANNOT_DECOMPRESS); -} - -void registerCodecLZSSE8(CompressionCodecFactory & factory) -{ - UInt8 method_code = UInt8(CompressionMethodByte::LZSSE8); - factory.registerCompressionCodec( - "LZSSE8", - method_code, - [&](const ASTPtr & arguments) -> CompressionCodecPtr - { - int level = 1; - if (arguments && !arguments->children.empty()) - { - if (arguments->children.size() != 1) - throw Exception( - "LZSSE8 codec must have 1 parameter, given " + std::to_string(arguments->children.size()), - ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE); - - const auto children = arguments->children; - const auto * level_literal = children[0]->as(); - if (!level_literal) - throw Exception("LZSSE8 first codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); - - level = level_literal->value.safeGet(); - - } - - return std::make_shared(level); - }); -} - -} diff --git a/src/Compression/CompressionCodecLizard.cpp b/src/Compression/CompressionCodecLizard.cpp index 2006a708493..17486693a7b 100644 --- a/src/Compression/CompressionCodecLizard.cpp +++ b/src/Compression/CompressionCodecLizard.cpp @@ -4,7 +4,6 @@ #include #include #include -#include namespace DB diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index a703040e98c..ebd30fea535 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -61,7 +61,8 @@ CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std } } -void CompressionCodecFactory::validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs) const +void CompressionCodecFactory::validateCodec( + const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs) const { if (family_name.empty()) throw Exception("Compression codec name cannot be empty", ErrorCodes::BAD_ARGUMENTS); @@ -69,12 +70,14 @@ void CompressionCodecFactory::validateCodec(const String & family_name, std::opt if (level) { auto literal = std::make_shared(static_cast(*level)); - validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), {}, sanity_check, allow_experimental_codecs); + validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), + {}, sanity_check, allow_experimental_codecs); } else { auto identifier = std::make_shared(Poco::toUpper(family_name)); - validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), {}, sanity_check, allow_experimental_codecs); + validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), + {}, sanity_check, allow_experimental_codecs); } } @@ -140,7 +143,8 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( if (column_type) { CompressionCodecPtr prev_codec; - IDataType::StreamCallbackWithType callback = [&](const ISerialization::SubstreamPath & substream_path, const IDataType & substream_type) + IDataType::StreamCallbackWithType callback = [&]( + const ISerialization::SubstreamPath & substream_path, const IDataType & substream_type) { if (ISerialization::isSpecialCompressionAllowed(substream_path)) { @@ -229,7 +233,9 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( throw Exception("Unknown codec family: " + queryToString(ast), ErrorCodes::UNKNOWN_CODEC); } -CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default, bool only_generic) const + +CompressionCodecPtr CompressionCodecFactory::get( + const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default, bool only_generic) const { if (current_default == nullptr) current_default = default_codec; @@ -280,6 +286,7 @@ CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, const IData throw Exception("Unexpected AST structure for compression codec: " + queryToString(ast), ErrorCodes::UNEXPECTED_AST_STRUCTURE); } + CompressionCodecPtr CompressionCodecFactory::get(const uint8_t byte_code) const { const auto family_code_and_creator = family_code_with_codec.find(byte_code); @@ -337,7 +344,7 @@ void CompressionCodecFactory::registerSimpleCompressionCodec( registerCompressionCodec(family_name, byte_code, [family_name, creator](const ASTPtr & ast) { if (ast) - throw Exception("Compression codec " + family_name + " cannot have arguments", ErrorCodes::DATA_TYPE_CANNOT_HAVE_ARGUMENTS); + throw Exception(ErrorCodes::DATA_TYPE_CANNOT_HAVE_ARGUMENTS, "Compression codec {} cannot have arguments", family_name); return creator(); }); } @@ -354,9 +361,7 @@ void registerCodecGorilla(CompressionCodecFactory & factory); void registerCodecMultiple(CompressionCodecFactory & factory); void registerCodecLizard(CompressionCodecFactory & factory); void registerCodecDensity(CompressionCodecFactory & factory); -void registerCodecLZSSE2(CompressionCodecFactory & factory); -void registerCodecLZSSE4(CompressionCodecFactory & factory); -void registerCodecLZSSE8(CompressionCodecFactory & factory); +void registerCodecsLZSSE(CompressionCodecFactory & factory); CompressionCodecFactory::CompressionCodecFactory() { @@ -371,9 +376,7 @@ CompressionCodecFactory::CompressionCodecFactory() registerCodecMultiple(*this); registerCodecLizard(*this); registerCodecDensity(*this); - registerCodecLZSSE2(*this); - registerCodecLZSSE4(*this); - registerCodecLZSSE8(*this); + registerCodecsLZSSE(*this); default_codec = get("LZ4", {}); } diff --git a/src/Compression/CompressionInfo.h b/src/Compression/CompressionInfo.h index d1d4ad4618f..44bed7d3c1d 100644 --- a/src/Compression/CompressionInfo.h +++ b/src/Compression/CompressionInfo.h @@ -47,7 +47,7 @@ enum class CompressionMethodByte : uint8_t Density = 0x97, LZSSE2 = 0x98, LZSSE4 = 0x99, - LZSSE8 = 0xa0, + LZSSE8 = 0xA0, }; diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index d08ffd730ac..4c8c8e60d69 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -3,8 +3,9 @@ #include #include #include -#include #include +#include + namespace DB { From deb68b15daeaa7e4279140cf4fa09bd1cdd154e3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 04:03:42 +0300 Subject: [PATCH 24/44] Fix bad dependencies in code --- src/Compression/CompressionCodecDensity.h | 2 +- src/Compression/CompressionCodecLZSSE.h | 1 + src/Compression/CompressionCodecLizard.h | 3 +-- src/Compression/CompressionFactory.cpp | 23 +++++++---------------- src/Compression/ICompressionCodec.h | 4 ++++ 5 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/Compression/CompressionCodecDensity.h b/src/Compression/CompressionCodecDensity.h index ffdbd8f3619..8b6f2a46109 100644 --- a/src/Compression/CompressionCodecDensity.h +++ b/src/Compression/CompressionCodecDensity.h @@ -25,8 +25,8 @@ protected: void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; bool isCompression() const override { return true; } - bool isGenericCompression() const override { return true; } + bool isExperimental() const override { return true; } private: const DENSITY_ALGORITHM algo; diff --git a/src/Compression/CompressionCodecLZSSE.h b/src/Compression/CompressionCodecLZSSE.h index 85cc59c7be0..8c32987f873 100644 --- a/src/Compression/CompressionCodecLZSSE.h +++ b/src/Compression/CompressionCodecLZSSE.h @@ -20,6 +20,7 @@ protected: void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; bool isCompression() const override { return true; } bool isGenericCompression() const override { return true; } + bool isExperimental() const override { return true; } private: const UInt32 type; diff --git a/src/Compression/CompressionCodecLizard.h b/src/Compression/CompressionCodecLizard.h index 8b39f224dbc..7304297a80c 100644 --- a/src/Compression/CompressionCodecLizard.h +++ b/src/Compression/CompressionCodecLizard.h @@ -20,12 +20,11 @@ public: protected: UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; bool isCompression() const override { return true; } - bool isGenericCompression() const override { return true; } + bool isExperimental() const override { return true; } private: const int level; diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index ebd30fea535..d73c94c8cc3 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -93,7 +93,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( std::optional generic_compression_codec_pos; bool can_substitute_codec_arguments = true; - for (size_t i = 0; i < func->arguments->children.size(); ++i) + for (size_t i = 0, size = func->arguments->children.size(); i < size; ++i) { const auto & inner_codec_ast = func->arguments->children[i]; String codec_family_name; @@ -111,21 +111,6 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( else throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - if (!allow_experimental_codecs) - { - if (codec_family_name == "Lizard" || - codec_family_name == "Density" || - codec_family_name == "LZSSE2" || - codec_family_name == "LZSSE4" || - codec_family_name == "LZSSE8") - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Codec {} is experimental and not meant to be used in production." - " You can enable it with the 'allow_experimental_codecs' setting.", - codec_family_name); - } - } - /// Default codec replaced with current default codec which may depend on different /// settings (and properties of data) in runtime. CompressionCodecPtr result_codec; @@ -169,6 +154,12 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( result_codec = getImpl(codec_family_name, codec_arguments, nullptr); } + if (!allow_experimental_codecs && result_codec->isExperimental()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Codec {} is experimental and not meant to be used in production." + " You can enable it with the 'allow_experimental_codecs' setting.", + codec_family_name); + codecs_descriptions->children.emplace_back(result_codec->getCodecDesc()); } diff --git a/src/Compression/ICompressionCodec.h b/src/Compression/ICompressionCodec.h index 2018218f2e7..47b4d9bfb43 100644 --- a/src/Compression/ICompressionCodec.h +++ b/src/Compression/ICompressionCodec.h @@ -73,6 +73,10 @@ public: /// Is it a generic compression algorithm like lz4, zstd. Usually it does not make sense to apply generic compression more than single time. virtual bool isGenericCompression() const = 0; + /// It is a codec available only for evaluation purposes and not meant to be used in production. + /// It will not be allowed to use unless the user will turn off the safety switch. + virtual bool isExperimental() const { return false; } + /// If it does nothing. virtual bool isNone() const { return false; } From 3057bbe8312b4a9c72ec82f9b87287d553066fbe Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 04:12:30 +0300 Subject: [PATCH 25/44] Remove header files --- src/Compression/CompressionCodecDelta.cpp | 25 +++- src/Compression/CompressionCodecDelta.h | 32 ----- src/Compression/CompressionCodecDensity.cpp | 32 ++++- src/Compression/CompressionCodecDensity.h | 35 ------ .../CompressionCodecDoubleDelta.cpp | 114 ++++++++++++++++- src/Compression/CompressionCodecDoubleDelta.h | 118 ------------------ src/Compression/CompressionCodecGorilla.cpp | 111 +++++++++++++++- src/Compression/CompressionCodecGorilla.h | 115 ----------------- src/Compression/CompressionCodecLZ4.cpp | 55 +++++++- src/Compression/CompressionCodecLZ4.h | 52 -------- src/Compression/CompressionCodecLZSSE.cpp | 25 +++- src/Compression/CompressionCodecLZSSE.h | 30 ----- src/Compression/CompressionCodecLizard.cpp | 28 ++++- src/Compression/CompressionCodecLizard.h | 33 ----- src/Compression/CompressionCodecT64.cpp | 57 ++++++++- src/Compression/CompressionCodecT64.h | 53 -------- src/Compression/CompressionCodecZSTD.cpp | 36 +++++- src/Compression/CompressionCodecZSTD.h | 42 ------- 18 files changed, 465 insertions(+), 528 deletions(-) delete mode 100644 src/Compression/CompressionCodecDelta.h delete mode 100644 src/Compression/CompressionCodecDensity.h delete mode 100644 src/Compression/CompressionCodecDoubleDelta.h delete mode 100644 src/Compression/CompressionCodecGorilla.h delete mode 100644 src/Compression/CompressionCodecLZ4.h delete mode 100644 src/Compression/CompressionCodecLZSSE.h delete mode 100644 src/Compression/CompressionCodecLizard.h delete mode 100644 src/Compression/CompressionCodecT64.h delete mode 100644 src/Compression/CompressionCodecZSTD.h diff --git a/src/Compression/CompressionCodecDelta.cpp b/src/Compression/CompressionCodecDelta.cpp index f3d6953bd2a..7d0824d4d71 100644 --- a/src/Compression/CompressionCodecDelta.cpp +++ b/src/Compression/CompressionCodecDelta.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -11,6 +11,29 @@ namespace DB { +class CompressionCodecDelta : public ICompressionCodec +{ +public: + CompressionCodecDelta(UInt8 delta_bytes_size_); + + uint8_t getMethodByte() const override; + + void updateHash(SipHash & hash) const override; + +protected: + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override { return uncompressed_size + 2; } + + bool isCompression() const override { return false; } + bool isGenericCompression() const override { return false; } + +private: + UInt8 delta_bytes_size; +}; + + namespace ErrorCodes { extern const int CANNOT_COMPRESS; diff --git a/src/Compression/CompressionCodecDelta.h b/src/Compression/CompressionCodecDelta.h deleted file mode 100644 index e892aa04242..00000000000 --- a/src/Compression/CompressionCodecDelta.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include - -namespace DB -{ - -class CompressionCodecDelta : public ICompressionCodec -{ -public: - CompressionCodecDelta(UInt8 delta_bytes_size_); - - uint8_t getMethodByte() const override; - - void updateHash(SipHash & hash) const override; - -protected: - - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override { return uncompressed_size + 2; } - - bool isCompression() const override { return false; } - bool isGenericCompression() const override { return false; } - -private: - UInt8 delta_bytes_size; -}; - -} diff --git a/src/Compression/CompressionCodecDensity.cpp b/src/Compression/CompressionCodecDensity.cpp index bd1fb25e18f..cdee69005be 100644 --- a/src/Compression/CompressionCodecDensity.cpp +++ b/src/Compression/CompressionCodecDensity.cpp @@ -1,11 +1,41 @@ +#include #include #include -#include #include +#include + namespace DB { + +class CompressionCodecDensity : public ICompressionCodec +{ +public: + static constexpr auto DENSITY_DEFAULT_ALGO = DENSITY_ALGORITHM_CHAMELEON; // by default aim on speed + + CompressionCodecDensity(DENSITY_ALGORITHM algo_); + + uint8_t getMethodByte() const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; + + void updateHash(SipHash & hash) const override; + +protected: + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + + bool isCompression() const override { return true; } + bool isGenericCompression() const override { return true; } + bool isExperimental() const override { return true; } + +private: + const DENSITY_ALGORITHM algo; +}; + + namespace ErrorCodes { extern const int CANNOT_COMPRESS; diff --git a/src/Compression/CompressionCodecDensity.h b/src/Compression/CompressionCodecDensity.h deleted file mode 100644 index 8b6f2a46109..00000000000 --- a/src/Compression/CompressionCodecDensity.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include -#include - - -namespace DB -{ -class CompressionCodecDensity : public ICompressionCodec -{ -public: - static constexpr auto DENSITY_DEFAULT_ALGO = DENSITY_ALGORITHM_CHAMELEON; // by default aim on speed - - CompressionCodecDensity(DENSITY_ALGORITHM algo_); - - uint8_t getMethodByte() const override; - - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - - void updateHash(SipHash & hash) const override; - -protected: - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - - bool isCompression() const override { return true; } - bool isGenericCompression() const override { return true; } - bool isExperimental() const override { return true; } - -private: - const DENSITY_ALGORITHM algo; -}; - -} diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp index 6895a80264d..d992132c4f6 100644 --- a/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/src/Compression/CompressionCodecDoubleDelta.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -15,9 +15,121 @@ #include #include + namespace DB { +/** DoubleDelta column codec implementation. + * + * Based on Gorilla paper: http://www.vldb.org/pvldb/vol8/p1816-teller.pdf, which was extended + * to support 64bit types. The drawback is 1 extra bit for 32-byte wide deltas: 5-bit prefix + * instead of 4-bit prefix. + * + * This codec is best used against monotonic integer sequences with constant (or almost constant) + * stride, like event timestamp for some monitoring application. + * + * Given input sequence a: [a0, a1, ... an]: + * + * First, write number of items (sizeof(int32)*8 bits): n + * Then write first item as is (sizeof(a[0])*8 bits): a[0] + * Second item is written as delta (sizeof(a[0])*8 bits): a[1] - a[0] + * Loop over remaining items and calculate double delta: + * double_delta = a[i] - 2 * a[i - 1] + a[i - 2] + * Write it in compact binary form with `BitWriter` + * if double_delta == 0: + * write 1bit: 0 + * else if -63 < double_delta < 64: + * write 2 bit prefix: 10 + * write sign bit (1 if signed): x + * write 7-1 bits of abs(double_delta - 1): xxxxxx + * else if -255 < double_delta < 256: + * write 3 bit prefix: 110 + * write sign bit (1 if signed): x + * write 9-1 bits of abs(double_delta - 1): xxxxxxxx + * else if -2047 < double_delta < 2048: + * write 4 bit prefix: 1110 + * write sign bit (1 if signed): x + * write 12-1 bits of abs(double_delta - 1): xxxxxxxxxxx + * else if double_delta fits into 32-bit int: + * write 5 bit prefix: 11110 + * write sign bit (1 if signed): x + * write 32-1 bits of abs(double_delta - 1): xxxxxxxxxxx... + * else + * write 5 bit prefix: 11111 + * write sign bit (1 if signed): x + * write 64-1 bits of abs(double_delta - 1): xxxxxxxxxxx... + * + * @example sequence of UInt8 values [1, 2, 3, 4, 5, 6, 7, 8, 9 10] is encoded as (codec header is omitted): + * + * .- 4-byte little-endian sequence length (10 == 0xa) + * | .- 1 byte (sizeof(UInt8) a[0] : 0x01 + * | | .- 1 byte of delta: a[1] - a[0] = 2 - 1 = 1 : 0x01 + * | | | .- 8 zero bits since double delta for remaining 8 elements was 0 : 0x00 + * v_______________v___v___v___ + * \x0a\x00\x00\x00\x01\x01\x00 + * + * @example sequence of Int16 values [-10, 10, -20, 20, -40, 40] is encoded as: + * + * .- 4-byte little endian sequence length = 6 : 0x00000006 + * | .- 2 bytes (sizeof(Int16) a[0] as UInt16 = -10 : 0xfff6 + * | | .- 2 bytes of delta: a[1] - a[0] = 10 - (-10) = 20 : 0x0014 + * | | | .- 4 encoded double deltas (see below) + * v_______________ v______ v______ v______________________ + * \x06\x00\x00\x00\xf6\xff\x14\x00\xb8\xe2\x2e\xb1\xe4\x58 + * + * 4 binary encoded double deltas (\xb8\xe2\x2e\xb1\xe4\x58): + * double_delta (DD) = -20 - 2 * 10 + (-10) = -50 + * .- 2-bit prefix : 0b10 + * | .- sign-bit : 0b1 + * | |.- abs(DD - 1) = 49 : 0b110001 + * | || + * | || DD = 20 - 2 * (-20) + 10 = 70 + * | || .- 3-bit prefix : 0b110 + * | || | .- sign bit : 0b0 + * | || | |.- abs(DD - 1) = 69 : 0b1000101 + * | || | || + * | || | || DD = -40 - 2 * 20 + (-20) = -100 + * | || | || .- 3-bit prefix : 0b110 + * | || | || | .- sign-bit : 0b0 + * | || | || | |.- abs(DD - 1) = 99 : 0b1100011 + * | || | || | || + * | || | || | || DD = 40 - 2 * (-40) + 20 = 140 + * | || | || | || .- 3-bit prefix : 0b110 + * | || | || | || | .- sign bit : 0b0 + * | || | || | || | |.- abs(DD - 1) = 139 : 0b10001011 + * | || | || | || | || + * V_vv______V__vv________V____vv_______V__vv________,- padding bits + * 10111000 11100010 00101110 10110001 11100100 01011000 + * + * Please also see unit tests for: + * * Examples on what output `BitWriter` produces on predefined input. + * * Compatibility tests solidifying encoded binary output on set of predefined sequences. + */ +class CompressionCodecDoubleDelta : public ICompressionCodec +{ +public: + CompressionCodecDoubleDelta(UInt8 data_bytes_size_); + + uint8_t getMethodByte() const override; + + void updateHash(SipHash & hash) const override; + +protected: + + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; + + bool isCompression() const override { return true; } + bool isGenericCompression() const override { return false; } + +private: + UInt8 data_bytes_size; +}; + + namespace ErrorCodes { extern const int CANNOT_COMPRESS; diff --git a/src/Compression/CompressionCodecDoubleDelta.h b/src/Compression/CompressionCodecDoubleDelta.h deleted file mode 100644 index 11140ded61e..00000000000 --- a/src/Compression/CompressionCodecDoubleDelta.h +++ /dev/null @@ -1,118 +0,0 @@ -#pragma once - -#include - -namespace DB -{ - -/** DoubleDelta column codec implementation. - * - * Based on Gorilla paper: http://www.vldb.org/pvldb/vol8/p1816-teller.pdf, which was extended - * to support 64bit types. The drawback is 1 extra bit for 32-byte wide deltas: 5-bit prefix - * instead of 4-bit prefix. - * - * This codec is best used against monotonic integer sequences with constant (or almost constant) - * stride, like event timestamp for some monitoring application. - * - * Given input sequence a: [a0, a1, ... an]: - * - * First, write number of items (sizeof(int32)*8 bits): n - * Then write first item as is (sizeof(a[0])*8 bits): a[0] - * Second item is written as delta (sizeof(a[0])*8 bits): a[1] - a[0] - * Loop over remaining items and calculate double delta: - * double_delta = a[i] - 2 * a[i - 1] + a[i - 2] - * Write it in compact binary form with `BitWriter` - * if double_delta == 0: - * write 1bit: 0 - * else if -63 < double_delta < 64: - * write 2 bit prefix: 10 - * write sign bit (1 if signed): x - * write 7-1 bits of abs(double_delta - 1): xxxxxx - * else if -255 < double_delta < 256: - * write 3 bit prefix: 110 - * write sign bit (1 if signed): x - * write 9-1 bits of abs(double_delta - 1): xxxxxxxx - * else if -2047 < double_delta < 2048: - * write 4 bit prefix: 1110 - * write sign bit (1 if signed): x - * write 12-1 bits of abs(double_delta - 1): xxxxxxxxxxx - * else if double_delta fits into 32-bit int: - * write 5 bit prefix: 11110 - * write sign bit (1 if signed): x - * write 32-1 bits of abs(double_delta - 1): xxxxxxxxxxx... - * else - * write 5 bit prefix: 11111 - * write sign bit (1 if signed): x - * write 64-1 bits of abs(double_delta - 1): xxxxxxxxxxx... - * - * @example sequence of UInt8 values [1, 2, 3, 4, 5, 6, 7, 8, 9 10] is encoded as (codec header is omitted): - * - * .- 4-byte little-endian sequence length (10 == 0xa) - * | .- 1 byte (sizeof(UInt8) a[0] : 0x01 - * | | .- 1 byte of delta: a[1] - a[0] = 2 - 1 = 1 : 0x01 - * | | | .- 8 zero bits since double delta for remaining 8 elements was 0 : 0x00 - * v_______________v___v___v___ - * \x0a\x00\x00\x00\x01\x01\x00 - * - * @example sequence of Int16 values [-10, 10, -20, 20, -40, 40] is encoded as: - * - * .- 4-byte little endian sequence length = 6 : 0x00000006 - * | .- 2 bytes (sizeof(Int16) a[0] as UInt16 = -10 : 0xfff6 - * | | .- 2 bytes of delta: a[1] - a[0] = 10 - (-10) = 20 : 0x0014 - * | | | .- 4 encoded double deltas (see below) - * v_______________ v______ v______ v______________________ - * \x06\x00\x00\x00\xf6\xff\x14\x00\xb8\xe2\x2e\xb1\xe4\x58 - * - * 4 binary encoded double deltas (\xb8\xe2\x2e\xb1\xe4\x58): - * double_delta (DD) = -20 - 2 * 10 + (-10) = -50 - * .- 2-bit prefix : 0b10 - * | .- sign-bit : 0b1 - * | |.- abs(DD - 1) = 49 : 0b110001 - * | || - * | || DD = 20 - 2 * (-20) + 10 = 70 - * | || .- 3-bit prefix : 0b110 - * | || | .- sign bit : 0b0 - * | || | |.- abs(DD - 1) = 69 : 0b1000101 - * | || | || - * | || | || DD = -40 - 2 * 20 + (-20) = -100 - * | || | || .- 3-bit prefix : 0b110 - * | || | || | .- sign-bit : 0b0 - * | || | || | |.- abs(DD - 1) = 99 : 0b1100011 - * | || | || | || - * | || | || | || DD = 40 - 2 * (-40) + 20 = 140 - * | || | || | || .- 3-bit prefix : 0b110 - * | || | || | || | .- sign bit : 0b0 - * | || | || | || | |.- abs(DD - 1) = 139 : 0b10001011 - * | || | || | || | || - * V_vv______V__vv________V____vv_______V__vv________,- padding bits - * 10111000 11100010 00101110 10110001 11100100 01011000 - * - * Please also see unit tests for: - * * Examples on what output `BitWriter` produces on predefined input. - * * Compatibility tests solidifying encoded binary output on set of predefined sequences. - */ -class CompressionCodecDoubleDelta : public ICompressionCodec -{ -public: - CompressionCodecDoubleDelta(UInt8 data_bytes_size_); - - uint8_t getMethodByte() const override; - - void updateHash(SipHash & hash) const override; - -protected: - - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - - bool isCompression() const override { return true; } - bool isGenericCompression() const override { return false; } - -private: - UInt8 data_bytes_size; -}; - -} diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index 582a9f41874..db76a854a97 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -14,9 +14,118 @@ #include + namespace DB { +/** Gorilla column codec implementation. + * + * Based on Gorilla paper: http://www.vldb.org/pvldb/vol8/p1816-teller.pdf + * + * This codec is best used against monotonic floating sequences, like CPU usage percentage + * or any other gauge. + * + * Given input sequence a: [a0, a1, ... an] + * + * First, write number of items (sizeof(int32)*8 bits): n + * Then write first item as is (sizeof(a[0])*8 bits): a[0] + * Loop over remaining items and calculate xor_diff: + * xor_diff = a[i] ^ a[i - 1] (e.g. 00000011'10110100) + * Write it in compact binary form with `BitWriter` + * if xor_diff == 0: + * write 1 bit: 0 + * else: + * calculate leading zero bits (lzb) + * and trailing zero bits (tzb) of xor_diff, + * compare to lzb and tzb of previous xor_diff + * (X = sizeof(a[i]) * 8, e.g. X = 16, lzb = 6, tzb = 2) + * if lzb >= prev_lzb && tzb >= prev_tzb: + * (e.g. prev_lzb=4, prev_tzb=1) + * write 2 bit prefix: 0b10 + * write xor_diff >> prev_tzb (X - prev_lzb - prev_tzb bits):0b00111011010 + * (where X = sizeof(a[i]) * 8, e.g. 16) + * else: + * write 2 bit prefix: 0b11 + * write 5 bits of lzb: 0b00110 + * write 6 bits of (X - lzb - tzb)=(16-6-2)=8: 0b001000 + * write (X - lzb - tzb) non-zero bits of xor_diff: 0b11101101 + * prev_lzb = lzb + * prev_tzb = tzb + * + * @example sequence of Float32 values [0.1, 0.1, 0.11, 0.2, 0.1] is encoded as: + * + * .- 4-byte little endian sequence length: 5 : 0x00000005 + * | .- 4 byte (sizeof(Float32) a[0] as UInt32 : -10 : 0xcdcccc3d + * | | .- 4 encoded xor diffs (see below) + * v_______________ v______________ v__________________________________________________ + * \x05\x00\x00\x00\xcd\xcc\xcc\x3d\x6a\x5a\xd8\xb6\x3c\xcd\x75\xb1\x6c\x77\x00\x00\x00 + * + * 4 binary encoded xor diffs (\x6a\x5a\xd8\xb6\x3c\xcd\x75\xb1\x6c\x77\x00\x00\x00): + * + * ........................................... + * a[i-1] = 00111101110011001100110011001101 + * a[i] = 00111101110011001100110011001101 + * xor_diff = 00000000000000000000000000000000 + * .- 1-bit prefix : 0b0 + * | + * | ........................................... + * | a[i-1] = 00111101110011001100110011001101 + * ! a[i] = 00111101111000010100011110101110 + * | xor_diff = 00000000001011011000101101100011 + * | lzb = 10 + * | tzb = 0 + * |.- 2-bit prefix : 0b11 + * || .- lzb (10) : 0b1010 + * || | .- data length (32-10-0): 22 : 0b010110 + * || | | .- data : 0b1011011000101101100011 + * || | | | + * || | | | ........................................... + * || | | | a[i-1] = 00111101111000010100011110101110 + * || | | | a[i] = 00111110010011001100110011001101 + * || | | | xor_diff = 00000011101011011000101101100011 + * || | | | .- 2-bit prefix : 0b11 + * || | | | | .- lzb = 6 : 0b00110 + * || | | | | | .- data length = (32 - 6) = 26 : 0b011010 + * || | | | | | | .- data : 0b11101011011000101101100011 + * || | | | | | | | + * || | | | | | | | ........................................... + * || | | | | | | | a[i-1] = 00111110010011001100110011001101 + * || | | | | | | | a[i] = 00111101110011001100110011001101 + * || | | | | | | | xor_diff = 00000011100000000000000000000000 + * || | | | | | | | .- 2-bit prefix : 0b10 + * || | | | | | | | | .- data : 0b11100000000000000000000000 + * VV_v____ v_____v________________________V_v_____v______v____________________________V_v_____________________________ + * 01101010 01011010 11011000 10110110 00111100 11001101 01110101 10110001 01101100 01110111 00000000 00000000 00000000 + * + * Please also see unit tests for: + * * Examples on what output `BitWriter` produces on predefined input. + * * Compatibility tests solidifying encoded binary output on set of predefined sequences. + */ +class CompressionCodecGorilla : public ICompressionCodec +{ +public: + CompressionCodecGorilla(UInt8 data_bytes_size_); + + uint8_t getMethodByte() const override; + + void updateHash(SipHash & hash) const override; + +protected: + + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; + + bool isCompression() const override { return true; } + bool isGenericCompression() const override { return false; } + +private: + UInt8 data_bytes_size; +}; + + namespace ErrorCodes { extern const int CANNOT_COMPRESS; diff --git a/src/Compression/CompressionCodecGorilla.h b/src/Compression/CompressionCodecGorilla.h deleted file mode 100644 index 3613ab2a96f..00000000000 --- a/src/Compression/CompressionCodecGorilla.h +++ /dev/null @@ -1,115 +0,0 @@ -#pragma once - -#include - -namespace DB -{ - -/** Gorilla column codec implementation. - * - * Based on Gorilla paper: http://www.vldb.org/pvldb/vol8/p1816-teller.pdf - * - * This codec is best used against monotonic floating sequences, like CPU usage percentage - * or any other gauge. - * - * Given input sequence a: [a0, a1, ... an] - * - * First, write number of items (sizeof(int32)*8 bits): n - * Then write first item as is (sizeof(a[0])*8 bits): a[0] - * Loop over remaining items and calculate xor_diff: - * xor_diff = a[i] ^ a[i - 1] (e.g. 00000011'10110100) - * Write it in compact binary form with `BitWriter` - * if xor_diff == 0: - * write 1 bit: 0 - * else: - * calculate leading zero bits (lzb) - * and trailing zero bits (tzb) of xor_diff, - * compare to lzb and tzb of previous xor_diff - * (X = sizeof(a[i]) * 8, e.g. X = 16, lzb = 6, tzb = 2) - * if lzb >= prev_lzb && tzb >= prev_tzb: - * (e.g. prev_lzb=4, prev_tzb=1) - * write 2 bit prefix: 0b10 - * write xor_diff >> prev_tzb (X - prev_lzb - prev_tzb bits):0b00111011010 - * (where X = sizeof(a[i]) * 8, e.g. 16) - * else: - * write 2 bit prefix: 0b11 - * write 5 bits of lzb: 0b00110 - * write 6 bits of (X - lzb - tzb)=(16-6-2)=8: 0b001000 - * write (X - lzb - tzb) non-zero bits of xor_diff: 0b11101101 - * prev_lzb = lzb - * prev_tzb = tzb - * - * @example sequence of Float32 values [0.1, 0.1, 0.11, 0.2, 0.1] is encoded as: - * - * .- 4-byte little endian sequence length: 5 : 0x00000005 - * | .- 4 byte (sizeof(Float32) a[0] as UInt32 : -10 : 0xcdcccc3d - * | | .- 4 encoded xor diffs (see below) - * v_______________ v______________ v__________________________________________________ - * \x05\x00\x00\x00\xcd\xcc\xcc\x3d\x6a\x5a\xd8\xb6\x3c\xcd\x75\xb1\x6c\x77\x00\x00\x00 - * - * 4 binary encoded xor diffs (\x6a\x5a\xd8\xb6\x3c\xcd\x75\xb1\x6c\x77\x00\x00\x00): - * - * ........................................... - * a[i-1] = 00111101110011001100110011001101 - * a[i] = 00111101110011001100110011001101 - * xor_diff = 00000000000000000000000000000000 - * .- 1-bit prefix : 0b0 - * | - * | ........................................... - * | a[i-1] = 00111101110011001100110011001101 - * ! a[i] = 00111101111000010100011110101110 - * | xor_diff = 00000000001011011000101101100011 - * | lzb = 10 - * | tzb = 0 - * |.- 2-bit prefix : 0b11 - * || .- lzb (10) : 0b1010 - * || | .- data length (32-10-0): 22 : 0b010110 - * || | | .- data : 0b1011011000101101100011 - * || | | | - * || | | | ........................................... - * || | | | a[i-1] = 00111101111000010100011110101110 - * || | | | a[i] = 00111110010011001100110011001101 - * || | | | xor_diff = 00000011101011011000101101100011 - * || | | | .- 2-bit prefix : 0b11 - * || | | | | .- lzb = 6 : 0b00110 - * || | | | | | .- data length = (32 - 6) = 26 : 0b011010 - * || | | | | | | .- data : 0b11101011011000101101100011 - * || | | | | | | | - * || | | | | | | | ........................................... - * || | | | | | | | a[i-1] = 00111110010011001100110011001101 - * || | | | | | | | a[i] = 00111101110011001100110011001101 - * || | | | | | | | xor_diff = 00000011100000000000000000000000 - * || | | | | | | | .- 2-bit prefix : 0b10 - * || | | | | | | | | .- data : 0b11100000000000000000000000 - * VV_v____ v_____v________________________V_v_____v______v____________________________V_v_____________________________ - * 01101010 01011010 11011000 10110110 00111100 11001101 01110101 10110001 01101100 01110111 00000000 00000000 00000000 - * - * Please also see unit tests for: - * * Examples on what output `BitWriter` produces on predefined input. - * * Compatibility tests solidifying encoded binary output on set of predefined sequences. - */ -class CompressionCodecGorilla : public ICompressionCodec -{ -public: - CompressionCodecGorilla(UInt8 data_bytes_size_); - - uint8_t getMethodByte() const override; - - void updateHash(SipHash & hash) const override; - -protected: - - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - - bool isCompression() const override { return true; } - bool isGenericCompression() const override { return false; } - -private: - UInt8 data_bytes_size; -}; - -} diff --git a/src/Compression/CompressionCodecLZ4.cpp b/src/Compression/CompressionCodecLZ4.cpp index 5f43b49706f..597a6c195eb 100644 --- a/src/Compression/CompressionCodecLZ4.cpp +++ b/src/Compression/CompressionCodecLZ4.cpp @@ -1,7 +1,7 @@ -#include "CompressionCodecLZ4.h" - #include #include + +#include #include #include #include @@ -9,7 +9,12 @@ #include #include #include +#include #include +#include + +#include + #pragma GCC diagnostic ignored "-Wold-style-cast" @@ -17,11 +22,51 @@ namespace DB { +class CompressionCodecLZ4 : public ICompressionCodec +{ +public: + CompressionCodecLZ4(); + + uint8_t getMethodByte() const override; + + UInt32 getAdditionalSizeAtTheEndOfBuffer() const override { return LZ4::ADDITIONAL_BYTES_AT_END_OF_BUFFER; } + + void updateHash(SipHash & hash) const override; + +protected: + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + + bool isCompression() const override { return true; } + bool isGenericCompression() const override { return true; } + +private: + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; + + mutable LZ4::PerformanceStatistics lz4_stat; + ASTPtr codec_desc; +}; + + +class CompressionCodecLZ4HC : public CompressionCodecLZ4 +{ +public: + CompressionCodecLZ4HC(int level_); + +protected: + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + +private: + const int level; +}; + + namespace ErrorCodes { -extern const int CANNOT_COMPRESS; -extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; -extern const int ILLEGAL_CODEC_PARAMETER; + extern const int CANNOT_COMPRESS; + extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; + extern const int ILLEGAL_CODEC_PARAMETER; } CompressionCodecLZ4::CompressionCodecLZ4() diff --git a/src/Compression/CompressionCodecLZ4.h b/src/Compression/CompressionCodecLZ4.h deleted file mode 100644 index bf8b4e2dd1f..00000000000 --- a/src/Compression/CompressionCodecLZ4.h +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -class CompressionCodecLZ4 : public ICompressionCodec -{ -public: - CompressionCodecLZ4(); - - uint8_t getMethodByte() const override; - - UInt32 getAdditionalSizeAtTheEndOfBuffer() const override { return LZ4::ADDITIONAL_BYTES_AT_END_OF_BUFFER; } - - void updateHash(SipHash & hash) const override; - -protected: - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - - bool isCompression() const override { return true; } - bool isGenericCompression() const override { return true; } - -private: - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - - mutable LZ4::PerformanceStatistics lz4_stat; - ASTPtr codec_desc; -}; - - -class CompressionCodecLZ4HC : public CompressionCodecLZ4 -{ -public: - CompressionCodecLZ4HC(int level_); - -protected: - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - -private: - const int level; -}; - -} diff --git a/src/Compression/CompressionCodecLZSSE.cpp b/src/Compression/CompressionCodecLZSSE.cpp index 05c5f507e24..50585766b34 100644 --- a/src/Compression/CompressionCodecLZSSE.cpp +++ b/src/Compression/CompressionCodecLZSSE.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -10,6 +10,29 @@ namespace DB { + +class CompressionCodecLZSSE : public ICompressionCodec +{ +public: + CompressionCodecLZSSE(UInt32 type_, UInt32 level_); + + uint8_t getMethodByte() const override; + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; + void updateHash(SipHash & hash) const override; + +protected: + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + bool isCompression() const override { return true; } + bool isGenericCompression() const override { return true; } + bool isExperimental() const override { return true; } + +private: + const UInt32 type; + const UInt32 level; +}; + + namespace ErrorCodes { extern const int CANNOT_COMPRESS; diff --git a/src/Compression/CompressionCodecLZSSE.h b/src/Compression/CompressionCodecLZSSE.h deleted file mode 100644 index 8c32987f873..00000000000 --- a/src/Compression/CompressionCodecLZSSE.h +++ /dev/null @@ -1,30 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ - -class CompressionCodecLZSSE : public ICompressionCodec -{ -public: - CompressionCodecLZSSE(UInt32 type_, UInt32 level_); - - uint8_t getMethodByte() const override; - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - void updateHash(SipHash & hash) const override; - -protected: - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - bool isCompression() const override { return true; } - bool isGenericCompression() const override { return true; } - bool isExperimental() const override { return true; } - -private: - const UInt32 type; - const UInt32 level; -}; - -} diff --git a/src/Compression/CompressionCodecLizard.cpp b/src/Compression/CompressionCodecLizard.cpp index 17486693a7b..2059b12bab1 100644 --- a/src/Compression/CompressionCodecLizard.cpp +++ b/src/Compression/CompressionCodecLizard.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -8,6 +8,32 @@ namespace DB { +class CompressionCodecLizard : public ICompressionCodec +{ +public: + static constexpr auto LIZARD_DEFAULT_LEVEL = 1; + + CompressionCodecLizard(int level_); + + uint8_t getMethodByte() const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; + + void updateHash(SipHash & hash) const override; + +protected: + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + + bool isCompression() const override { return true; } + bool isGenericCompression() const override { return true; } + bool isExperimental() const override { return true; } + +private: + const int level; +}; + + namespace ErrorCodes { extern const int CANNOT_COMPRESS; diff --git a/src/Compression/CompressionCodecLizard.h b/src/Compression/CompressionCodecLizard.h deleted file mode 100644 index 7304297a80c..00000000000 --- a/src/Compression/CompressionCodecLizard.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ -class CompressionCodecLizard : public ICompressionCodec -{ -public: - static constexpr auto LIZARD_DEFAULT_LEVEL = 1; - - CompressionCodecLizard(int level_); - - uint8_t getMethodByte() const override; - - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - - void updateHash(SipHash & hash) const override; - -protected: - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - - bool isCompression() const override { return true; } - bool isGenericCompression() const override { return true; } - bool isExperimental() const override { return true; } - -private: - const int level; -}; - -} diff --git a/src/Compression/CompressionCodecT64.cpp b/src/Compression/CompressionCodecT64.cpp index f0fb8351410..377db915bce 100644 --- a/src/Compression/CompressionCodecT64.cpp +++ b/src/Compression/CompressionCodecT64.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include #include @@ -8,18 +8,63 @@ #include #include #include +#include namespace DB { +/// Get 64 integer values, makes 64x64 bit matrix, transpose it and crop unused bits (most significant zeroes). +/// In example, if we have UInt8 with only 0 and 1 inside 64xUInt8 would be compressed into 1xUInt64. +/// It detects unused bits by calculating min and max values of data part, saving them in header in compression phase. +/// There's a special case with signed integers parts with crossing zero data. Here it stores one more bit to detect sign of value. +class CompressionCodecT64 : public ICompressionCodec +{ +public: + static constexpr UInt32 HEADER_SIZE = 1 + 2 * sizeof(UInt64); + static constexpr UInt32 MAX_COMPRESSED_BLOCK_SIZE = sizeof(UInt64) * 64; + + /// There're 2 compression variants: + /// Byte - transpose bit matrix by bytes (only the last not full byte is transposed by bits). It's default. + /// Bits - full bit-transpose of the bit matrix. It uses more resources and leads to better compression with ZSTD (but worse with LZ4). + enum class Variant + { + Byte, + Bit + }; + + CompressionCodecT64(TypeIndex type_idx_, Variant variant_); + + uint8_t getMethodByte() const override; + + void updateHash(SipHash & hash) const override; + +protected: + UInt32 doCompressData(const char * src, UInt32 src_size, char * dst) const override; + void doDecompressData(const char * src, UInt32 src_size, char * dst, UInt32 uncompressed_size) const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override + { + /// uncompressed_size - (uncompressed_size % (sizeof(T) * 64)) + sizeof(UInt64) * sizeof(T) + header_size + return uncompressed_size + MAX_COMPRESSED_BLOCK_SIZE + HEADER_SIZE; + } + + bool isCompression() const override { return true; } + bool isGenericCompression() const override { return false; } + +private: + TypeIndex type_idx; + Variant variant; +}; + + namespace ErrorCodes { -extern const int CANNOT_COMPRESS; -extern const int CANNOT_DECOMPRESS; -extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; -extern const int ILLEGAL_CODEC_PARAMETER; -extern const int LOGICAL_ERROR; + extern const int CANNOT_COMPRESS; + extern const int CANNOT_DECOMPRESS; + extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; + extern const int ILLEGAL_CODEC_PARAMETER; + extern const int LOGICAL_ERROR; } namespace diff --git a/src/Compression/CompressionCodecT64.h b/src/Compression/CompressionCodecT64.h deleted file mode 100644 index c7664679d5a..00000000000 --- a/src/Compression/CompressionCodecT64.h +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once - -#include -#include - - -namespace DB -{ - -/// Get 64 integer values, makes 64x64 bit matrix, transpose it and crop unused bits (most significant zeroes). -/// In example, if we have UInt8 with only 0 and 1 inside 64xUInt8 would be compressed into 1xUInt64. -/// It detects unused bits by calculating min and max values of data part, saving them in header in compression phase. -/// There's a special case with signed integers parts with crossing zero data. Here it stores one more bit to detect sign of value. -class CompressionCodecT64 : public ICompressionCodec -{ -public: - static constexpr UInt32 HEADER_SIZE = 1 + 2 * sizeof(UInt64); - static constexpr UInt32 MAX_COMPRESSED_BLOCK_SIZE = sizeof(UInt64) * 64; - - /// There're 2 compression variants: - /// Byte - transpose bit matrix by bytes (only the last not full byte is transposed by bits). It's default. - /// Bits - full bit-transpose of the bit matrix. It uses more resources and leads to better compression with ZSTD (but worse with LZ4). - enum class Variant - { - Byte, - Bit - }; - - CompressionCodecT64(TypeIndex type_idx_, Variant variant_); - - uint8_t getMethodByte() const override; - - void updateHash(SipHash & hash) const override; - -protected: - UInt32 doCompressData(const char * src, UInt32 src_size, char * dst) const override; - void doDecompressData(const char * src, UInt32 src_size, char * dst, UInt32 uncompressed_size) const override; - - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override - { - /// uncompressed_size - (uncompressed_size % (sizeof(T) * 64)) + sizeof(UInt64) * sizeof(T) + header_size - return uncompressed_size + MAX_COMPRESSED_BLOCK_SIZE + HEADER_SIZE; - } - - bool isCompression() const override { return true; } - bool isGenericCompression() const override { return false; } - -private: - TypeIndex type_idx; - Variant variant; -}; - -} diff --git a/src/Compression/CompressionCodecZSTD.cpp b/src/Compression/CompressionCodecZSTD.cpp index 378f74ce7cf..4d3b5a181d1 100644 --- a/src/Compression/CompressionCodecZSTD.cpp +++ b/src/Compression/CompressionCodecZSTD.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -7,11 +7,45 @@ #include #include #include +#include +#include namespace DB { +class CompressionCodecZSTD : public ICompressionCodec +{ +public: + static constexpr auto ZSTD_DEFAULT_LEVEL = 1; + static constexpr auto ZSTD_DEFAULT_LOG_WINDOW = 24; + + CompressionCodecZSTD(int level_); + + CompressionCodecZSTD(int level_, int window_log); + + uint8_t getMethodByte() const override; + + UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; + + void updateHash(SipHash & hash) const override; + +protected: + + UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; + + void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; + + bool isCompression() const override { return true; } + bool isGenericCompression() const override { return true; } + +private: + const int level; + const bool enable_long_range; + const int window_log; +}; + + namespace ErrorCodes { extern const int CANNOT_COMPRESS; diff --git a/src/Compression/CompressionCodecZSTD.h b/src/Compression/CompressionCodecZSTD.h deleted file mode 100644 index 2d67ff9cecb..00000000000 --- a/src/Compression/CompressionCodecZSTD.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace DB -{ - -class CompressionCodecZSTD : public ICompressionCodec -{ -public: - static constexpr auto ZSTD_DEFAULT_LEVEL = 1; - static constexpr auto ZSTD_DEFAULT_LOG_WINDOW = 24; - - CompressionCodecZSTD(int level_); - - CompressionCodecZSTD(int level_, int window_log); - - uint8_t getMethodByte() const override; - - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - - void updateHash(SipHash & hash) const override; - -protected: - - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - - bool isCompression() const override { return true; } - bool isGenericCompression() const override { return true; } - -private: - const int level; - const bool enable_long_range; - const int window_log; -}; - -} From fdc5bb72f54aa78534047201fc10644d1bb14d81 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 04:14:29 +0300 Subject: [PATCH 26/44] Minor modification --- src/Compression/CompressionFactory.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index d73c94c8cc3..8965b499ea1 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -13,6 +13,7 @@ namespace DB { + namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -49,14 +50,16 @@ CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std if (level && param) { auto level_literal = std::make_shared(static_cast(*level)); - auto param_literal = std::make_shared(static_cast(*param)); + auto param_literal = std::make_shared(*param); return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), level_literal, param_literal)), {}); } else if (param) { - auto param_literal = std::make_shared(static_cast(*param)); + auto param_literal = std::make_shared(*param); return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), param_literal)), {}); - } else { + } + else + { return get(family_name, level); } } From 40d4f0678f0ea24ff97b10bb090b4122aa507198 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 May 2021 04:25:06 +0300 Subject: [PATCH 27/44] Remove overload (harmful) --- programs/compressor/Compressor.cpp | 2 +- src/Client/Connection.cpp | 2 +- src/Compression/CompressionFactory.cpp | 24 +++++++------------ src/Compression/CompressionFactory.h | 2 -- src/DataStreams/TemporaryFileStream.h | 2 +- src/Server/TCPHandler.cpp | 2 +- src/Storages/CompressionCodecSelector.h | 2 +- .../DistributedBlockOutputStream.cpp | 2 +- src/Storages/MergeTree/DataPartsExchange.cpp | 4 ++-- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 4 ++-- .../MergeTree/MergeTreeDataPartInMemory.cpp | 4 ++-- .../MergeTree/MergeTreeWriteAheadLog.cpp | 7 +++++- 12 files changed, 26 insertions(+), 31 deletions(-) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index 9f99f8eb95e..520fff199f0 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -144,7 +144,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) codec = CompressionCodecFactory::instance().get(ast, nullptr); } else - codec = CompressionCodecFactory::instance().get(method_family, level); + codec = CompressionCodecFactory::instance().get(method_family, level, {}); std::unique_ptr rb; diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 87f768d7e75..4346510269c 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -425,7 +425,7 @@ void Connection::sendQuery( level = settings->network_zstd_compression_level; CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs); - compression_codec = CompressionCodecFactory::instance().get(method, level); + compression_codec = CompressionCodecFactory::instance().get(method, level, {}); } else compression_codec = CompressionCodecFactory::instance().getDefaultCodec(); diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index 8965b499ea1..1ce8c82fdd0 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -31,20 +31,6 @@ CompressionCodecPtr CompressionCodecFactory::getDefaultCodec() const } -CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std::optional level) const -{ - if (level) - { - auto literal = std::make_shared(static_cast(*level)); - return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), {}); - } - else - { - auto identifier = std::make_shared(Poco::toUpper(family_name)); - return get(makeASTFunction("CODEC", identifier), {}); - } -} - CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std::optional level, std::optional param) const { if (level && param) @@ -58,9 +44,15 @@ CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std auto param_literal = std::make_shared(*param); return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), param_literal)), {}); } + else if (level) + { + auto level_literal = std::make_shared(static_cast(*level)); + return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), level_literal)), {}); + } else { - return get(family_name, level); + auto identifier = std::make_shared(Poco::toUpper(family_name)); + return get(makeASTFunction("CODEC", identifier), {}); } } @@ -372,7 +364,7 @@ CompressionCodecFactory::CompressionCodecFactory() registerCodecDensity(*this); registerCodecsLZSSE(*this); - default_codec = get("LZ4", {}); + default_codec = get("LZ4", {}, {}); } CompressionCodecFactory & CompressionCodecFactory::instance() diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h index 34dd8633611..a616b06c285 100644 --- a/src/Compression/CompressionFactory.h +++ b/src/Compression/CompressionFactory.h @@ -70,8 +70,6 @@ public: CompressionCodecPtr get(const uint8_t byte_code) const; /// For backward compatibility with config settings - CompressionCodecPtr get(const String & family_name, std::optional level) const; - CompressionCodecPtr get(const String & family_name, std::optional level, std::optional param) const; /// Register codec with parameters and column type diff --git a/src/DataStreams/TemporaryFileStream.h b/src/DataStreams/TemporaryFileStream.h index ce9071801d0..ac366106897 100644 --- a/src/DataStreams/TemporaryFileStream.h +++ b/src/DataStreams/TemporaryFileStream.h @@ -36,7 +36,7 @@ struct TemporaryFileStream std::atomic * is_cancelled, const std::string & codec) { WriteBufferFromFile file_buf(path); - CompressedWriteBuffer compressed_buf(file_buf, CompressionCodecFactory::instance().get(codec, {})); + CompressedWriteBuffer compressed_buf(file_buf, CompressionCodecFactory::instance().get(codec, {}, {})); NativeBlockOutputStream output(compressed_buf, 0, header); copyData(input, output, is_cancelled); compressed_buf.finalize(); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index e147083d3b7..4baa3c455d2 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1401,7 +1401,7 @@ void TCPHandler::initBlockOutput(const Block & block) CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs); state.maybe_compressed_out = std::make_shared( - *out, CompressionCodecFactory::instance().get(method, level)); + *out, CompressionCodecFactory::instance().get(method, level, {})); } else state.maybe_compressed_out = out; diff --git a/src/Storages/CompressionCodecSelector.h b/src/Storages/CompressionCodecSelector.h index 746b3ce37ee..76c88546547 100644 --- a/src/Storages/CompressionCodecSelector.h +++ b/src/Storages/CompressionCodecSelector.h @@ -92,7 +92,7 @@ public: for (const auto & element : elements) if (element.check(part_size, part_size_ratio)) - res = factory.get(element.family_name, element.level); + res = factory.get(element.family_name, element.level, {}); return res; } diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index b91f5b10cf2..ed40290c0d3 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -633,7 +633,7 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: compression_level = settings.network_zstd_compression_level; CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs); - CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level); + CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level, {}); /// tmp directory is used to ensure atomicity of transactions /// and keep monitor thread out from reading incomplete data diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index d5f4d91850c..f290d6780d5 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -600,7 +600,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( metadata_snapshot->projections.get(projection_name).metadata, block.getNamesAndTypesList(), {}, - CompressionCodecFactory::instance().get("NONE", {})); + CompressionCodecFactory::instance().get("NONE", {}, {})); part_out.writePrefix(); part_out.write(block); part_out.writeSuffixAndFinalizePart(new_projection_part); @@ -622,7 +622,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( new_data_part->partition.create(metadata_snapshot, block, 0); MergedBlockOutputStream part_out( - new_data_part, metadata_snapshot, block.getNamesAndTypesList(), {}, CompressionCodecFactory::instance().get("NONE", {})); + new_data_part, metadata_snapshot, block.getNamesAndTypesList(), {}, CompressionCodecFactory::instance().get("NONE", {}, {})); part_out.writePrefix(); part_out.write(block); part_out.writeSuffixAndFinalizePart(new_data_part); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index dc68a1dcb41..9fac317bfbd 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -691,7 +691,7 @@ void IMergeTreeDataPart::loadDefaultCompressionCodec() /// In memory parts doesn't have any compression if (!isStoredOnDisk()) { - default_codec = CompressionCodecFactory::instance().get("NONE", {}); + default_codec = CompressionCodecFactory::instance().get("NONE", {}, {}); return; } @@ -733,7 +733,7 @@ CompressionCodecPtr IMergeTreeDataPart::detectDefaultCompressionCodec() const { /// In memory parts doesn't have any compression if (!isStoredOnDisk()) - return CompressionCodecFactory::instance().get("NONE", {}); + return CompressionCodecFactory::instance().get("NONE", {}, {}); auto metadata_snapshot = storage.getInMemoryMetadataPtr(); diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp index 473f2e598a9..555b514bd38 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp @@ -26,7 +26,7 @@ MergeTreeDataPartInMemory::MergeTreeDataPartInMemory( const IMergeTreeDataPart * parent_part_) : IMergeTreeDataPart(storage_, name_, volume_, relative_path_, Type::IN_MEMORY, parent_part_) { - default_codec = CompressionCodecFactory::instance().get("NONE", {}); + default_codec = CompressionCodecFactory::instance().get("NONE", {}, {}); } MergeTreeDataPartInMemory::MergeTreeDataPartInMemory( @@ -38,7 +38,7 @@ MergeTreeDataPartInMemory::MergeTreeDataPartInMemory( const IMergeTreeDataPart * parent_part_) : IMergeTreeDataPart(storage_, name_, info_, volume_, relative_path_, Type::IN_MEMORY, parent_part_) { - default_codec = CompressionCodecFactory::instance().get("NONE", {}); + default_codec = CompressionCodecFactory::instance().get("NONE", {}, {}); } IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartInMemory::getReader( diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp index 4c92d4f6136..bd5412b0bdf 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp @@ -189,7 +189,12 @@ MergeTreeData::MutableDataPartsVector MergeTreeWriteAheadLog::restore(const Stor if (action_type == ActionType::ADD_PART) { - MergedBlockOutputStream part_out(part, metadata_snapshot, block.getNamesAndTypesList(), {}, CompressionCodecFactory::instance().get("NONE", {})); + MergedBlockOutputStream part_out( + part, + metadata_snapshot, + block.getNamesAndTypesList(), + {}, + CompressionCodecFactory::instance().get("NONE", {}, {})); part->minmax_idx.update(block, storage.getMinMaxColumnsNames(metadata_snapshot->getPartitionKey())); part->partition.create(metadata_snapshot, block, 0); From 408b828777f4bb6f037e7a12338b4da89ef12689 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 00:48:11 +0300 Subject: [PATCH 28/44] Fix typo --- programs/compressor/Compressor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index 520fff199f0..99ae4b3039f 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -81,7 +81,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) ("block-size,b", po::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") ("hc", "use LZ4HC instead of LZ4") ("zstd", "use ZSTD instead of LZ4") - ("param", po::value(), "extra params for compresion algorithm") + ("param", po::value(), "extra params for compression algorithm") ("codec", po::value>()->multitoken(), "use codecs combination instead of LZ4") ("level", po::value(), "compression level for codecs specified via flags") ("none", "use no compression instead of LZ4") From 57a3f132d7ad2a764c804b81233454eed57fce9f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 00:48:54 +0300 Subject: [PATCH 29/44] Fix style --- src/Compression/CompressionCodecLZ4.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Compression/CompressionCodecLZ4.cpp b/src/Compression/CompressionCodecLZ4.cpp index 597a6c195eb..5bf6b48258c 100644 --- a/src/Compression/CompressionCodecLZ4.cpp +++ b/src/Compression/CompressionCodecLZ4.cpp @@ -13,9 +13,6 @@ #include #include -#include - - #pragma GCC diagnostic ignored "-Wold-style-cast" From 8d2fa6cf50cb9dd78e34b248da2bc65c311efca5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 01:18:12 +0300 Subject: [PATCH 30/44] Allow to build without experimental compression libraries and remove from fasttest --- CMakeLists.txt | 4 +++- cmake/find/density.cmake | 9 +++++++++ cmake/find/lizard.cmake | 9 +++++++++ cmake/find/lzsse.cmake | 9 +++++++++ cmake/find/yaml-cpp.cmake | 2 +- contrib/CMakeLists.txt | 15 +++++++++++---- docker/test/fasttest/run.sh | 3 --- src/Common/config.h.in | 5 ++++- src/Compression/CompressionCodecDensity.cpp | 8 ++++++++ src/Compression/CompressionCodecLZSSE.cpp | 8 ++++++++ src/Compression/CompressionCodecLizard.cpp | 8 ++++++++ src/Compression/CompressionFactory.cpp | 19 +++++++++++++++++++ src/Core/config_core.h.in | 3 +++ 13 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 cmake/find/density.cmake create mode 100644 cmake/find/lizard.cmake create mode 100644 cmake/find/lzsse.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 866d9f542e1..cc016d4f2cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -528,7 +528,9 @@ include (cmake/find/rocksdb.cmake) include (cmake/find/libpqxx.cmake) include (cmake/find/nuraft.cmake) include (cmake/find/yaml-cpp.cmake) - +include (cmake/find/lizard.cmake) +include (cmake/find/lzsse.cmake) +include (cmake/find/density.cmake) if(NOT USE_INTERNAL_PARQUET_LIBRARY) set (ENABLE_ORC OFF CACHE INTERNAL "") diff --git a/cmake/find/density.cmake b/cmake/find/density.cmake new file mode 100644 index 00000000000..f4963d67709 --- /dev/null +++ b/cmake/find/density.cmake @@ -0,0 +1,9 @@ +option(USE_DENSITY "Enable DENSITY experimental compression library" ${ENABLE_LIBRARIES}) + +if (NOT USE_DENSITY) + return() +endif() + +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/density/README.md") + message (ERROR "submodule contrib/density is missing. to fix try run: \n git submodule update --init --recursive") +endif() diff --git a/cmake/find/lizard.cmake b/cmake/find/lizard.cmake new file mode 100644 index 00000000000..27c8945c45a --- /dev/null +++ b/cmake/find/lizard.cmake @@ -0,0 +1,9 @@ +option(USE_LIZARD "Enable Lizard experimental compression library" ${ENABLE_LIBRARIES}) + +if (NOT USE_LIZARD) + return() +endif() + +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/lizard/README.md") + message (ERROR "submodule contrib/lizard is missing. to fix try run: \n git submodule update --init --recursive") +endif() diff --git a/cmake/find/lzsse.cmake b/cmake/find/lzsse.cmake new file mode 100644 index 00000000000..a5f58a255f8 --- /dev/null +++ b/cmake/find/lzsse.cmake @@ -0,0 +1,9 @@ +option(USE_LZSSE "Enable LZSSE experimental compression library" ${ENABLE_LIBRARIES}) + +if (NOT USE_LZSSE) + return() +endif() + +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/lzsse/README.md") + message (ERROR "submodule contrib/lzsse is missing. to fix try run: \n git submodule update --init --recursive") +endif() diff --git a/cmake/find/yaml-cpp.cmake b/cmake/find/yaml-cpp.cmake index 9b9d9bd39d6..4633d559980 100644 --- a/cmake/find/yaml-cpp.cmake +++ b/cmake/find/yaml-cpp.cmake @@ -4,6 +4,6 @@ if (NOT USE_YAML_CPP) return() endif() -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp") +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp/README.md") message (ERROR "submodule contrib/yaml-cpp is missing. to fix try run: \n git submodule update --init --recursive") endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 67e4a5f1e24..94c8199db0d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -58,14 +58,21 @@ if (USE_INTERNAL_XZ_LIBRARY) add_subdirectory (xz) endif() -add_subdirectory (lizard-cmake) -add_subdirectory (density-cmake) -add_subdirectory (lzsse-cmake) +if (USE_LIZARD) + add_subdirectory (lizard-cmake) +endif () + +if (USE_DENSITY) + add_subdirectory (density-cmake) +endif () + +if (USE_LZSSE) + add_subdirectory (lzsse-cmake) +endif () add_subdirectory (poco-cmake) add_subdirectory (croaring-cmake) - # TODO: refactor the contrib libraries below this comment. if (USE_INTERNAL_ZSTD_LIBRARY) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index d94ed5c876a..3a19a249f8e 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -185,9 +185,6 @@ function clone_submodules contrib/dragonbox contrib/fast_float contrib/NuRaft - contrib/lizard - contrib/density - contrib/lzsse ) git submodule sync diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 6844f0fa9e3..7396e6f6222 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -14,6 +14,9 @@ #cmakedefine01 USE_SENTRY #cmakedefine01 USE_GRPC #cmakedefine01 USE_STATS -#cmakedefine01 CLICKHOUSE_SPLIT_BINARY #cmakedefine01 USE_DATASKETCHES #cmakedefine01 USE_YAML_CPP +#cmakedefine01 USE_LIZARD +#cmakedefine01 USE_DENSITY +#cmakedefine01 USE_LZSSE +#cmakedefine01 CLICKHOUSE_SPLIT_BINARY diff --git a/src/Compression/CompressionCodecDensity.cpp b/src/Compression/CompressionCodecDensity.cpp index cdee69005be..033f2627364 100644 --- a/src/Compression/CompressionCodecDensity.cpp +++ b/src/Compression/CompressionCodecDensity.cpp @@ -1,3 +1,9 @@ +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + +#if USE_DENSITY + #include #include #include @@ -118,3 +124,5 @@ void registerCodecDensity(CompressionCodecFactory & factory) } } + +#endif diff --git a/src/Compression/CompressionCodecLZSSE.cpp b/src/Compression/CompressionCodecLZSSE.cpp index 50585766b34..48465c52125 100644 --- a/src/Compression/CompressionCodecLZSSE.cpp +++ b/src/Compression/CompressionCodecLZSSE.cpp @@ -1,3 +1,9 @@ +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + +#if USE_LZSSE + #include #include #include @@ -150,3 +156,5 @@ void registerCodecsLZSSE(CompressionCodecFactory & factory) } } + +#endif diff --git a/src/Compression/CompressionCodecLizard.cpp b/src/Compression/CompressionCodecLizard.cpp index 2059b12bab1..bc6607c0da9 100644 --- a/src/Compression/CompressionCodecLizard.cpp +++ b/src/Compression/CompressionCodecLizard.cpp @@ -1,3 +1,9 @@ +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + +#if USE_LIZARD + #include #include #include @@ -111,3 +117,5 @@ void registerCodecLizard(CompressionCodecFactory & factory) } } + +#endif diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index 1ce8c82fdd0..db73e6df4df 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -1,3 +1,7 @@ +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + #include #include #include @@ -345,9 +349,18 @@ void registerCodecT64(CompressionCodecFactory & factory); void registerCodecDoubleDelta(CompressionCodecFactory & factory); void registerCodecGorilla(CompressionCodecFactory & factory); void registerCodecMultiple(CompressionCodecFactory & factory); + +#if USE_LIZARD void registerCodecLizard(CompressionCodecFactory & factory); +#endif + +#if USE_DENSITY void registerCodecDensity(CompressionCodecFactory & factory); +#endif + +#if USE_LZSSE void registerCodecsLZSSE(CompressionCodecFactory & factory); +#endif CompressionCodecFactory::CompressionCodecFactory() { @@ -360,9 +373,15 @@ CompressionCodecFactory::CompressionCodecFactory() registerCodecDoubleDelta(*this); registerCodecGorilla(*this); registerCodecMultiple(*this); +#if USE_LIZARD registerCodecLizard(*this); +#endif +#if USE_DENSITY registerCodecDensity(*this); +#endif +#if USE_LZSSE registerCodecsLZSSE(*this); +#endif default_codec = get("LZ4", {}, {}); } diff --git a/src/Core/config_core.h.in b/src/Core/config_core.h.in index e250e013913..da923ca23a1 100644 --- a/src/Core/config_core.h.in +++ b/src/Core/config_core.h.in @@ -15,3 +15,6 @@ #cmakedefine01 USE_LIBPQXX #cmakedefine01 USE_NURAFT #cmakedefine01 USE_KRB5 +#cmakedefine01 USE_LIZARD +#cmakedefine01 USE_DENSITY +#cmakedefine01 USE_LZSSE From 99b85bd032e6a64674b88a4b15768d23b82cc8d6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 01:34:59 +0300 Subject: [PATCH 31/44] Fix CMake --- src/CMakeLists.txt | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 25775d1f0c1..06aaa781292 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -379,28 +379,16 @@ if (XZ_LIBRARY) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${XZ_INCLUDE_DIR}) endif() -set (LIZARD_LIBRARY lizard) -set (LIZARD_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/lizard/lib/) -if (LIZARD_LIBRARY) - dbms_target_link_libraries(PRIVATE ${LIZARD_LIBRARY}) - #target_link_libraries (clickhouse_compression PUBLIC ${LZMA_LIBRARY}) - #target_include_directories (clickhouse_compression SYSTEM BEFORE PUBLIC ${}) +if (USE_LIZARD) + dbms_target_link_libraries(PRIVATE lizard) endif() -set (DENSITY_LIBRARY density) -set (DENSITY_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/density/src) -if (DENSITY_LIBRARY) - dbms_target_link_libraries(PRIVATE ${DENSITY_LIBRARY}) - #target_link_libraries (clickhouse_compression PUBLIC ${LZMA_LIBRARY}) - #target_include_directories (clickhouse_compression SYSTEM BEFORE PUBLIC ${}) +if (USE_DENSITY) + dbms_target_link_libraries(PRIVATE density) endif() -set (LZSSE_LIBRARY lzsse) -set (LZSSE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/lzsse/) -if (LZSSE_LIBRARY) - dbms_target_link_libraries(PRIVATE ${LZSSE_LIBRARY}) - #target_link_libraries (clickhouse_compression PUBLIC ${LZMA_LIBRARY}) - #target_include_directories (clickhouse_compression SYSTEM BEFORE PUBLIC ${}) +if (USE_LZSSE) + dbms_target_link_libraries(PRIVATE lzsse) endif() if (USE_ICU) From 70d9f5c1545041a118a9379fbdcdf7e18d8667ce Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 02:36:50 +0300 Subject: [PATCH 32/44] Fix error --- src/Compression/CompressionCodecLZSSE.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Compression/CompressionCodecLZSSE.cpp b/src/Compression/CompressionCodecLZSSE.cpp index 48465c52125..436f2dea1f2 100644 --- a/src/Compression/CompressionCodecLZSSE.cpp +++ b/src/Compression/CompressionCodecLZSSE.cpp @@ -92,16 +92,16 @@ UInt32 CompressionCodecLZSSE::doCompressData(const char * source, UInt32 source_ } case 4: { - LZSSE2_OptimalParseState * state = LZSSE2_MakeOptimalParseState(source_size); - res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level); - LZSSE2_FreeOptimalParseState(state); + LZSSE4_OptimalParseState * state = LZSSE4_MakeOptimalParseState(source_size); + res = LZSSE4_CompressOptimalParse(state, source, source_size, dest, source_size, level); + LZSSE4_FreeOptimalParseState(state); break; } case 8: { - LZSSE2_OptimalParseState * state = LZSSE2_MakeOptimalParseState(source_size); - res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level); - LZSSE2_FreeOptimalParseState(state); + LZSSE8_OptimalParseState * state = LZSSE8_MakeOptimalParseState(source_size); + res = LZSSE8_CompressOptimalParse(state, source, source_size, dest, source_size, level); + LZSSE8_FreeOptimalParseState(state); break; } default: From 541224290836c2dc040f633222597f96bd8d40c0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 02:38:46 +0300 Subject: [PATCH 33/44] Fix error --- src/Compression/CompressionCodecLZSSE.cpp | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/Compression/CompressionCodecLZSSE.cpp b/src/Compression/CompressionCodecLZSSE.cpp index 436f2dea1f2..1585f375eef 100644 --- a/src/Compression/CompressionCodecLZSSE.cpp +++ b/src/Compression/CompressionCodecLZSSE.cpp @@ -115,7 +115,27 @@ UInt32 CompressionCodecLZSSE::doCompressData(const char * source, UInt32 source_ void CompressionCodecLZSSE::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const { - UInt32 res = LZSSE2_Decompress(source, source_size, dest, uncompressed_size); + UInt32 res = 0; + switch (type) + { + case 2: + { + res = LZSSE2_Decompress(source, source_size, dest, uncompressed_size); + break; + } + case 4: + { + res = LZSSE4_Decompress(source, source_size, dest, uncompressed_size); + break; + } + case 8: + { + res = LZSSE8_Decompress(source, source_size, dest, uncompressed_size); + break; + } + default: + break; + } if (res < uncompressed_size) throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress block with LZSSE{}", type); } From ff7b5d34bba959d0dfc3063345949f91917dd8dd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 02:56:03 +0300 Subject: [PATCH 34/44] Mark false positives for PVS-Studio --- src/Common/tests/gtest_thread_pool_global_full.cpp | 4 ++-- src/Interpreters/ActionsDAG.cpp | 5 ++--- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- src/Parsers/New/ParserErrorListener.cpp | 2 +- src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 8 +++++--- src/Storages/MergeTree/MergeTreeDataSelectExecutor.h | 2 +- src/Storages/MergeTree/MergeTreeReaderWide.cpp | 2 +- src/Storages/SelectQueryInfo.h | 2 +- src/Storages/StorageLog.cpp | 2 +- 9 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/Common/tests/gtest_thread_pool_global_full.cpp b/src/Common/tests/gtest_thread_pool_global_full.cpp index 583d43be1bb..c6133f7ca11 100644 --- a/src/Common/tests/gtest_thread_pool_global_full.cpp +++ b/src/Common/tests/gtest_thread_pool_global_full.cpp @@ -25,7 +25,7 @@ TEST(ThreadPool, GlobalFull1) std::atomic counter = 0; static constexpr size_t num_jobs = capacity + 1; - auto func = [&] { ++counter; while (counter != num_jobs) {} }; + auto func = [&] { ++counter; while (counter != num_jobs) {} }; //-V776 ThreadPool pool(num_jobs); @@ -63,7 +63,7 @@ TEST(ThreadPool, GlobalFull2) global_pool.wait(); std::atomic counter = 0; - auto func = [&] { ++counter; while (counter != capacity + 1) {} }; + auto func = [&] { ++counter; while (counter != capacity + 1) {} }; //-V776 ThreadPool pool(capacity, 0, capacity); for (size_t i = 0; i < capacity; ++i) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 21c24956453..d1c10e1c037 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -540,7 +540,7 @@ Block ActionsDAG::updateHeader(Block header) const struct Frame { - const Node * node; + const Node * node = nullptr; size_t next_child = 0; }; @@ -587,8 +587,7 @@ Block ActionsDAG::updateHeader(Block header) const } } - auto & column = node_to_column[output]; - if (column.column) + if (node_to_column[output].column) result_columns.push_back(node_to_column[output]); } } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index cca29dd364d..9c463d89a59 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -263,7 +263,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) } /// We use global context here, because storages lifetime is bigger than query context lifetime - database->loadStoredObjects(getContext()->getGlobalContext(), has_force_restore_data_flag, create.attach && force_attach); + database->loadStoredObjects(getContext()->getGlobalContext(), has_force_restore_data_flag, create.attach && force_attach); //-V560 } catch (...) { diff --git a/src/Parsers/New/ParserErrorListener.cpp b/src/Parsers/New/ParserErrorListener.cpp index bc5ee84cba9..f6ac0f0c451 100644 --- a/src/Parsers/New/ParserErrorListener.cpp +++ b/src/Parsers/New/ParserErrorListener.cpp @@ -26,7 +26,7 @@ void ParserErrorListener::syntaxError( auto * parser = dynamic_cast(recognizer); assert(parser); - LOG_ERROR(&Poco::Logger::get("ClickHouseParser"), + LOG_ERROR(&Poco::Logger::get("ClickHouseParser"), //-V522 "Last element parsed so far:\n" "{}\n" "Parser error: (pos {}) {}", parser->getRuleContext()->toStringTree(parser, true), token->getStartIndex(), message); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 0f5e69448d2..c31585d1373 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -577,6 +577,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( MergeTreeDataSelectSamplingData sampling = use_cache ? std::move(cache->sampling) : MergeTreeDataSelectSamplingData{}; if (!use_cache) { + assert(key_condition.has_value()); + RelativeSize relative_sample_size = 0; RelativeSize relative_sample_offset = 0; @@ -605,7 +607,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( /// read) into the relative `SAMPLE 0.1` (how much data to read). size_t approx_total_rows = 0; if (relative_sample_size > 1 || relative_sample_offset > 1) - approx_total_rows = getApproximateTotalRowsToRead(parts, metadata_snapshot, *key_condition, settings); + approx_total_rows = getApproximateTotalRowsToRead(parts, metadata_snapshot, *key_condition, settings); //-V1007 if (relative_sample_size > 1) { @@ -764,7 +766,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( if (has_lower_limit) { - if (!key_condition->addCondition(sampling_key.column_names[0], Range::createLeftBounded(lower, true))) + if (!key_condition->addCondition(sampling_key.column_names[0], Range::createLeftBounded(lower, true))) //-V1007 throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN); ASTPtr args = std::make_shared(); @@ -781,7 +783,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( if (has_upper_limit) { - if (!key_condition->addCondition(sampling_key.column_names[0], Range::createRightBounded(upper, false))) + if (!key_condition->addCondition(sampling_key.column_names[0], Range::createRightBounded(upper, false))) //-V1007 throw Exception("Sampling column not in primary key", ErrorCodes::ILLEGAL_COLUMN); ASTPtr args = std::make_shared(); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 8335e54213f..70025b49658 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -15,7 +15,7 @@ class KeyCondition; struct MergeTreeDataSelectSamplingData { - bool use_sampling; + bool use_sampling = false; std::shared_ptr filter_function; ActionsDAGPtr filter_expression; }; diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 0da2f643eb0..f2357a87535 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -173,7 +173,7 @@ void MergeTreeReaderWide::readData( { auto get_stream_getter = [&](bool stream_for_prefix) -> ISerialization::InputStreamGetter { - return [&, stream_for_prefix](const ISerialization::SubstreamPath & substream_path) -> ReadBuffer * + return [&, stream_for_prefix](const ISerialization::SubstreamPath & substream_path) -> ReadBuffer * //-V1047 { /// If substream have already been read. if (cache.count(ISerialization::getSubcolumnNameForStream(substream_path))) diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index e971e126972..afed41189c2 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -122,7 +122,7 @@ struct MergeTreeDataSelectCache; // The projection selected to execute current query struct ProjectionCandidate { - const ProjectionDescription * desc; + const ProjectionDescription * desc{}; PrewhereInfoPtr prewhere_info; ActionsDAGPtr before_where; String where_column_name; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 663deddfd1f..61fbbbc3086 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -172,7 +172,7 @@ void LogSource::readData(const NameAndTypePair & name_and_type, ColumnPtr & colu auto create_stream_getter = [&](bool stream_for_prefix) { - return [&, stream_for_prefix] (const ISerialization::SubstreamPath & path) -> ReadBuffer * + return [&, stream_for_prefix] (const ISerialization::SubstreamPath & path) -> ReadBuffer * //-V1047 { if (cache.count(ISerialization::getSubcolumnNameForStream(path))) return nullptr; From 4290b48a6169784a64f80cc63f6c437f547fcabb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 06:00:57 +0300 Subject: [PATCH 35/44] Fix clang-tidy --- src/Compression/CompressionCodecDelta.cpp | 2 +- src/Compression/CompressionCodecDensity.cpp | 2 +- src/Compression/CompressionCodecDoubleDelta.cpp | 7 ++++++- src/Compression/CompressionCodecGorilla.cpp | 2 +- src/Compression/CompressionCodecLZ4.cpp | 4 ++-- src/Compression/CompressionCodecLZSSE.cpp | 2 +- src/Compression/CompressionCodecLizard.cpp | 2 +- src/Compression/CompressionCodecZSTD.cpp | 3 +-- 8 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/Compression/CompressionCodecDelta.cpp b/src/Compression/CompressionCodecDelta.cpp index 7d0824d4d71..447abe9e840 100644 --- a/src/Compression/CompressionCodecDelta.cpp +++ b/src/Compression/CompressionCodecDelta.cpp @@ -14,7 +14,7 @@ namespace DB class CompressionCodecDelta : public ICompressionCodec { public: - CompressionCodecDelta(UInt8 delta_bytes_size_); + explicit CompressionCodecDelta(UInt8 delta_bytes_size_); uint8_t getMethodByte() const override; diff --git a/src/Compression/CompressionCodecDensity.cpp b/src/Compression/CompressionCodecDensity.cpp index 033f2627364..88d5a294c81 100644 --- a/src/Compression/CompressionCodecDensity.cpp +++ b/src/Compression/CompressionCodecDensity.cpp @@ -20,7 +20,7 @@ class CompressionCodecDensity : public ICompressionCodec public: static constexpr auto DENSITY_DEFAULT_ALGO = DENSITY_ALGORITHM_CHAMELEON; // by default aim on speed - CompressionCodecDensity(DENSITY_ALGORITHM algo_); + explicit CompressionCodecDensity(DENSITY_ALGORITHM algo_); uint8_t getMethodByte() const override; diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp index d992132c4f6..79ced55594a 100644 --- a/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/src/Compression/CompressionCodecDoubleDelta.cpp @@ -19,6 +19,11 @@ namespace DB { +/** NOTE DoubleDelta is surprisingly bad name. The only excuse is that it comes from an academic paper. + * Most people will think that "double delta" is just applying delta transform twice. + * But in fact it is something more than applying delta transform twice. + */ + /** DoubleDelta column codec implementation. * * Based on Gorilla paper: http://www.vldb.org/pvldb/vol8/p1816-teller.pdf, which was extended @@ -108,7 +113,7 @@ namespace DB class CompressionCodecDoubleDelta : public ICompressionCodec { public: - CompressionCodecDoubleDelta(UInt8 data_bytes_size_); + explicit CompressionCodecDoubleDelta(UInt8 data_bytes_size_); uint8_t getMethodByte() const override; diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index db76a854a97..7fcb2183503 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -104,7 +104,7 @@ namespace DB class CompressionCodecGorilla : public ICompressionCodec { public: - CompressionCodecGorilla(UInt8 data_bytes_size_); + explicit CompressionCodecGorilla(UInt8 data_bytes_size_); uint8_t getMethodByte() const override; diff --git a/src/Compression/CompressionCodecLZ4.cpp b/src/Compression/CompressionCodecLZ4.cpp index 5bf6b48258c..8cb81e460b1 100644 --- a/src/Compression/CompressionCodecLZ4.cpp +++ b/src/Compression/CompressionCodecLZ4.cpp @@ -22,7 +22,7 @@ namespace DB class CompressionCodecLZ4 : public ICompressionCodec { public: - CompressionCodecLZ4(); + explicit CompressionCodecLZ4(); uint8_t getMethodByte() const override; @@ -49,7 +49,7 @@ private: class CompressionCodecLZ4HC : public CompressionCodecLZ4 { public: - CompressionCodecLZ4HC(int level_); + explicit CompressionCodecLZ4HC(int level_); protected: UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; diff --git a/src/Compression/CompressionCodecLZSSE.cpp b/src/Compression/CompressionCodecLZSSE.cpp index 1585f375eef..62fb641bc12 100644 --- a/src/Compression/CompressionCodecLZSSE.cpp +++ b/src/Compression/CompressionCodecLZSSE.cpp @@ -20,7 +20,7 @@ namespace DB class CompressionCodecLZSSE : public ICompressionCodec { public: - CompressionCodecLZSSE(UInt32 type_, UInt32 level_); + explicit CompressionCodecLZSSE(UInt32 type_, UInt32 level_); uint8_t getMethodByte() const override; UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; diff --git a/src/Compression/CompressionCodecLizard.cpp b/src/Compression/CompressionCodecLizard.cpp index bc6607c0da9..cd2d49a0538 100644 --- a/src/Compression/CompressionCodecLizard.cpp +++ b/src/Compression/CompressionCodecLizard.cpp @@ -19,7 +19,7 @@ class CompressionCodecLizard : public ICompressionCodec public: static constexpr auto LIZARD_DEFAULT_LEVEL = 1; - CompressionCodecLizard(int level_); + explicit CompressionCodecLizard(int level_); uint8_t getMethodByte() const override; diff --git a/src/Compression/CompressionCodecZSTD.cpp b/src/Compression/CompressionCodecZSTD.cpp index 4d3b5a181d1..a950f1fee74 100644 --- a/src/Compression/CompressionCodecZSTD.cpp +++ b/src/Compression/CompressionCodecZSTD.cpp @@ -20,8 +20,7 @@ public: static constexpr auto ZSTD_DEFAULT_LEVEL = 1; static constexpr auto ZSTD_DEFAULT_LOG_WINDOW = 24; - CompressionCodecZSTD(int level_); - + explicit CompressionCodecZSTD(int level_); CompressionCodecZSTD(int level_, int window_log); uint8_t getMethodByte() const override; From ad9481b1bbb94ee41c84532ae1ec1d1250fc3ea9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 06:04:13 +0300 Subject: [PATCH 36/44] Disable LZSSE on AArch64 --- cmake/find/lzsse.cmake | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmake/find/lzsse.cmake b/cmake/find/lzsse.cmake index a5f58a255f8..763483d8ea9 100644 --- a/cmake/find/lzsse.cmake +++ b/cmake/find/lzsse.cmake @@ -1,4 +1,9 @@ -option(USE_LZSSE "Enable LZSSE experimental compression library" ${ENABLE_LIBRARIES}) +set (DEFAULT_USE_LZSSE 0) +if (ENABLE_LIBRARIES AND ARCH_AMD64) + set (DEFAULT_USE_LZSSE 1) +endif() + +option(USE_LZSSE "Enable LZSSE experimental compression library" ${DEFAULT_USE_LZSSE}) if (NOT USE_LZSSE) return() From e206959390fbd6080bf1a06e8dbb76c30d05fa88 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 06:18:57 +0300 Subject: [PATCH 37/44] Use entropy coders from zstd --- contrib/lizard-cmake/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/lizard-cmake/CMakeLists.txt b/contrib/lizard-cmake/CMakeLists.txt index ad5f5bf132d..7fe5274ecb6 100644 --- a/contrib/lizard-cmake/CMakeLists.txt +++ b/contrib/lizard-cmake/CMakeLists.txt @@ -14,3 +14,6 @@ SET (Headers ADD_LIBRARY(lizard ${Sources} ${Headers}) target_include_directories (lizard PUBLIC ${LIBRARY_DIR}) + +# It is using some symbols (HUF_* and FSE_*) from zstd. +target_link_libraries (lizard PRIVATE zstd) From 273226de32b6880d403012935e0e8a16d7ba542e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 06:43:25 +0300 Subject: [PATCH 38/44] Remove string parameter for Density --- programs/compressor/Compressor.cpp | 2 +- src/Client/Connection.cpp | 2 +- src/Compression/CompressionCodecDensity.cpp | 24 +++++++------------ src/Compression/CompressionFactory.cpp | 17 +++---------- src/Compression/CompressionFactory.h | 2 +- src/DataStreams/TemporaryFileStream.h | 2 +- src/Server/TCPHandler.cpp | 2 +- src/Storages/CompressionCodecSelector.h | 2 +- .../DistributedBlockOutputStream.cpp | 2 +- src/Storages/MergeTree/DataPartsExchange.cpp | 4 ++-- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 4 ++-- .../MergeTree/MergeTreeDataPartInMemory.cpp | 4 ++-- .../MergeTree/MergeTreeWriteAheadLog.cpp | 2 +- .../test_non_default_compression/test.py | 2 +- 14 files changed, 27 insertions(+), 44 deletions(-) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index 99ae4b3039f..35534eaee9e 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -144,7 +144,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) codec = CompressionCodecFactory::instance().get(ast, nullptr); } else - codec = CompressionCodecFactory::instance().get(method_family, level, {}); + codec = CompressionCodecFactory::instance().get(method_family, level); std::unique_ptr rb; diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 4346510269c..87f768d7e75 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -425,7 +425,7 @@ void Connection::sendQuery( level = settings->network_zstd_compression_level; CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs); - compression_codec = CompressionCodecFactory::instance().get(method, level, {}); + compression_codec = CompressionCodecFactory::instance().get(method, level); } else compression_codec = CompressionCodecFactory::instance().getDefaultCodec(); diff --git a/src/Compression/CompressionCodecDensity.cpp b/src/Compression/CompressionCodecDensity.cpp index 88d5a294c81..e462ff2765b 100644 --- a/src/Compression/CompressionCodecDensity.cpp +++ b/src/Compression/CompressionCodecDensity.cpp @@ -18,8 +18,6 @@ namespace DB class CompressionCodecDensity : public ICompressionCodec { public: - static constexpr auto DENSITY_DEFAULT_ALGO = DENSITY_ALGORITHM_CHAMELEON; // by default aim on speed - explicit CompressionCodecDensity(DENSITY_ALGORITHM algo_); uint8_t getMethodByte() const override; @@ -93,7 +91,7 @@ void registerCodecDensity(CompressionCodecFactory & factory) method_code, [&](const ASTPtr & arguments) -> CompressionCodecPtr { - DENSITY_ALGORITHM algo = CompressionCodecDensity::DENSITY_DEFAULT_ALGO; + DENSITY_ALGORITHM algorithm = DENSITY_ALGORITHM_CHAMELEON; if (arguments && !arguments->children.empty()) { @@ -104,22 +102,18 @@ void registerCodecDensity(CompressionCodecFactory & factory) const auto children = arguments->children; const auto * algo_literal = children[0]->as(); - if (!algo_literal || algo_literal->value.getType() != Field::Types::String) - throw Exception("Density codec argument must be string (algorithm), one of: 'lion', 'chameleon', 'cheetah'", + if (!algo_literal || algo_literal->value.getType() != Field::Types::UInt64) + throw Exception("Density codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); - const auto algorithm = algo_literal->value.safeGet(); - if (algorithm == "lion") - algo = DENSITY_ALGORITHM_LION; - else if (algorithm == "chameleon") - algo = DENSITY_ALGORITHM_CHAMELEON; - else if (algorithm == "cheetah") - algo = DENSITY_ALGORITHM_CHEETAH; - else - throw Exception("Density codec argument may be one of: 'lion', 'chameleon', 'cheetah'", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + const UInt64 algorithm_num = algo_literal->value.safeGet(); + if (algorithm_num < 1 || algorithm_num > 3) + throw Exception("Density codec level can be 1, 2 or 3.", ErrorCodes::ILLEGAL_CODEC_PARAMETER); + + algorithm = static_cast(algorithm_num); } - return std::make_shared(algo); + return std::make_shared(algorithm); }); } diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index db73e6df4df..fbd369fdfe7 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -35,20 +35,9 @@ CompressionCodecPtr CompressionCodecFactory::getDefaultCodec() const } -CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std::optional level, std::optional param) const +CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std::optional level) const { - if (level && param) - { - auto level_literal = std::make_shared(static_cast(*level)); - auto param_literal = std::make_shared(*param); - return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), level_literal, param_literal)), {}); - } - else if (param) - { - auto param_literal = std::make_shared(*param); - return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), param_literal)), {}); - } - else if (level) + if (level) { auto level_literal = std::make_shared(static_cast(*level)); return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), level_literal)), {}); @@ -383,7 +372,7 @@ CompressionCodecFactory::CompressionCodecFactory() registerCodecsLZSSE(*this); #endif - default_codec = get("LZ4", {}, {}); + default_codec = get("LZ4", {}); } CompressionCodecFactory & CompressionCodecFactory::instance() diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h index a616b06c285..cdbb663935a 100644 --- a/src/Compression/CompressionFactory.h +++ b/src/Compression/CompressionFactory.h @@ -70,7 +70,7 @@ public: CompressionCodecPtr get(const uint8_t byte_code) const; /// For backward compatibility with config settings - CompressionCodecPtr get(const String & family_name, std::optional level, std::optional param) const; + CompressionCodecPtr get(const String & family_name, std::optional level) const; /// Register codec with parameters and column type void registerCompressionCodecWithType(const String & family_name, std::optional byte_code, CreatorWithType creator); diff --git a/src/DataStreams/TemporaryFileStream.h b/src/DataStreams/TemporaryFileStream.h index ac366106897..ce9071801d0 100644 --- a/src/DataStreams/TemporaryFileStream.h +++ b/src/DataStreams/TemporaryFileStream.h @@ -36,7 +36,7 @@ struct TemporaryFileStream std::atomic * is_cancelled, const std::string & codec) { WriteBufferFromFile file_buf(path); - CompressedWriteBuffer compressed_buf(file_buf, CompressionCodecFactory::instance().get(codec, {}, {})); + CompressedWriteBuffer compressed_buf(file_buf, CompressionCodecFactory::instance().get(codec, {})); NativeBlockOutputStream output(compressed_buf, 0, header); copyData(input, output, is_cancelled); compressed_buf.finalize(); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 4baa3c455d2..e147083d3b7 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1401,7 +1401,7 @@ void TCPHandler::initBlockOutput(const Block & block) CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs); state.maybe_compressed_out = std::make_shared( - *out, CompressionCodecFactory::instance().get(method, level, {})); + *out, CompressionCodecFactory::instance().get(method, level)); } else state.maybe_compressed_out = out; diff --git a/src/Storages/CompressionCodecSelector.h b/src/Storages/CompressionCodecSelector.h index 76c88546547..746b3ce37ee 100644 --- a/src/Storages/CompressionCodecSelector.h +++ b/src/Storages/CompressionCodecSelector.h @@ -92,7 +92,7 @@ public: for (const auto & element : elements) if (element.check(part_size, part_size_ratio)) - res = factory.get(element.family_name, element.level, {}); + res = factory.get(element.family_name, element.level); return res; } diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index ed40290c0d3..b91f5b10cf2 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -633,7 +633,7 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: compression_level = settings.network_zstd_compression_level; CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs); - CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level, {}); + CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level); /// tmp directory is used to ensure atomicity of transactions /// and keep monitor thread out from reading incomplete data diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index f290d6780d5..d5f4d91850c 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -600,7 +600,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( metadata_snapshot->projections.get(projection_name).metadata, block.getNamesAndTypesList(), {}, - CompressionCodecFactory::instance().get("NONE", {}, {})); + CompressionCodecFactory::instance().get("NONE", {})); part_out.writePrefix(); part_out.write(block); part_out.writeSuffixAndFinalizePart(new_projection_part); @@ -622,7 +622,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( new_data_part->partition.create(metadata_snapshot, block, 0); MergedBlockOutputStream part_out( - new_data_part, metadata_snapshot, block.getNamesAndTypesList(), {}, CompressionCodecFactory::instance().get("NONE", {}, {})); + new_data_part, metadata_snapshot, block.getNamesAndTypesList(), {}, CompressionCodecFactory::instance().get("NONE", {})); part_out.writePrefix(); part_out.write(block); part_out.writeSuffixAndFinalizePart(new_data_part); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 9fac317bfbd..dc68a1dcb41 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -691,7 +691,7 @@ void IMergeTreeDataPart::loadDefaultCompressionCodec() /// In memory parts doesn't have any compression if (!isStoredOnDisk()) { - default_codec = CompressionCodecFactory::instance().get("NONE", {}, {}); + default_codec = CompressionCodecFactory::instance().get("NONE", {}); return; } @@ -733,7 +733,7 @@ CompressionCodecPtr IMergeTreeDataPart::detectDefaultCompressionCodec() const { /// In memory parts doesn't have any compression if (!isStoredOnDisk()) - return CompressionCodecFactory::instance().get("NONE", {}, {}); + return CompressionCodecFactory::instance().get("NONE", {}); auto metadata_snapshot = storage.getInMemoryMetadataPtr(); diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp index 555b514bd38..473f2e598a9 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp @@ -26,7 +26,7 @@ MergeTreeDataPartInMemory::MergeTreeDataPartInMemory( const IMergeTreeDataPart * parent_part_) : IMergeTreeDataPart(storage_, name_, volume_, relative_path_, Type::IN_MEMORY, parent_part_) { - default_codec = CompressionCodecFactory::instance().get("NONE", {}, {}); + default_codec = CompressionCodecFactory::instance().get("NONE", {}); } MergeTreeDataPartInMemory::MergeTreeDataPartInMemory( @@ -38,7 +38,7 @@ MergeTreeDataPartInMemory::MergeTreeDataPartInMemory( const IMergeTreeDataPart * parent_part_) : IMergeTreeDataPart(storage_, name_, info_, volume_, relative_path_, Type::IN_MEMORY, parent_part_) { - default_codec = CompressionCodecFactory::instance().get("NONE", {}, {}); + default_codec = CompressionCodecFactory::instance().get("NONE", {}); } IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartInMemory::getReader( diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp index bd5412b0bdf..c8ae051b041 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp @@ -194,7 +194,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeWriteAheadLog::restore(const Stor metadata_snapshot, block.getNamesAndTypesList(), {}, - CompressionCodecFactory::instance().get("NONE", {}, {})); + CompressionCodecFactory::instance().get("NONE", {})); part->minmax_idx.update(block, storage.getMinMaxColumnsNames(metadata_snapshot->getPartitionKey())); part->partition.create(metadata_snapshot, block, 0); diff --git a/tests/integration/test_non_default_compression/test.py b/tests/integration/test_non_default_compression/test.py index 9689b4110a2..54e14132d3b 100644 --- a/tests/integration/test_non_default_compression/test.py +++ b/tests/integration/test_non_default_compression/test.py @@ -143,7 +143,7 @@ def test_experimental_codecs(start_cluster): node6.query(""" CREATE TABLE compression_experimental_codecs ( somedate Date CODEC(Lizard(12)), - id UInt64 CODEC(Density('lion')), + id UInt64 CODEC(Density(3)), data String CODEC(LZSSE4(3)) ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; """) From 54f0591aa5b400073b10325f774d97373c7c7909 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 06:47:00 +0300 Subject: [PATCH 39/44] Allow to ATTACH a table with experimental codecs --- src/Interpreters/InterpreterCreateQuery.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 9c463d89a59..8bc74e5b82d 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -447,7 +447,8 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( defaults_sample_block = validateColumnsDefaultsAndGetSampleBlock(default_expr_list, column_names_and_types, context_); bool sanity_check_compression_codecs = !attach && !context_->getSettingsRef().allow_suspicious_codecs; - bool allow_experimental_codecs = context_->getSettingsRef().allow_experimental_codecs; + bool allow_experimental_codecs = attach || context_->getSettingsRef().allow_experimental_codecs; + ColumnsDescription res; auto name_type_it = column_names_and_types.begin(); for (auto ast_it = columns_ast.children.begin(); ast_it != columns_ast.children.end(); ++ast_it, ++name_type_it) From 890b6e5e47fb5dbb12b163c58a37d292da1899cd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 06:54:42 +0300 Subject: [PATCH 40/44] Add a test --- .../00096_experimental_codecs.reference | 8 +++ .../1_stateful/00096_experimental_codecs.sql | 61 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 tests/queries/1_stateful/00096_experimental_codecs.reference create mode 100644 tests/queries/1_stateful/00096_experimental_codecs.sql diff --git a/tests/queries/1_stateful/00096_experimental_codecs.reference b/tests/queries/1_stateful/00096_experimental_codecs.reference new file mode 100644 index 00000000000..a34182c23e5 --- /dev/null +++ b/tests/queries/1_stateful/00096_experimental_codecs.reference @@ -0,0 +1,8 @@ +17415138241754778329 +17415138241754778329 +17415138241754778329 +17415138241754778329 +17415138241754778329 +3798944011853532000 +3798944011853532000 +3798944011853532000 diff --git a/tests/queries/1_stateful/00096_experimental_codecs.sql b/tests/queries/1_stateful/00096_experimental_codecs.sql new file mode 100644 index 00000000000..e085e55f886 --- /dev/null +++ b/tests/queries/1_stateful/00096_experimental_codecs.sql @@ -0,0 +1,61 @@ +DROP TABLE IF EXISTS hits_experimental; + +-- It's not allowed to create a table with experimental codecs unless the user turns off the safety switch. +CREATE TABLE hits_experimental (Title String CODEC(Lizard(10))) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 } + +SET allow_experimental_codecs = 1; + +-- Lizard + +CREATE TABLE hits_experimental (Title String CODEC(Lizard(10))) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO hits_experimental SELECT Title FROM test.hits; +SELECT sum(cityHash64(*)) FROM hits_experimental; + +-- It's always allowed to attach a table with experimental codecs. +DETACH TABLE hits_experimental; +SET allow_experimental_codecs = 0; +ATTACH TABLE hits_experimental; +SELECT sum(cityHash64(*)) FROM hits_experimental; +SET allow_experimental_codecs = 1; + +DROP TABLE hits_experimental; + +-- Density + +-- Check out of range levels of Density. +CREATE TABLE hits_experimental (Title String CODEC(Density(-1))) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 433 } +CREATE TABLE hits_experimental (Title String CODEC(Density(0))) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 433 } +CREATE TABLE hits_experimental (Title String CODEC(Density(4))) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 433 } +CREATE TABLE hits_experimental (Title String CODEC(Density('hello'))) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 433 } + +CREATE TABLE hits_experimental (Title String CODEC(Density(1))) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO hits_experimental SELECT Title FROM test.hits; +SELECT sum(cityHash64(*)) FROM hits_experimental; +DROP TABLE hits_experimental; + +CREATE TABLE hits_experimental (Title String CODEC(Density(2))) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO hits_experimental SELECT Title FROM test.hits; +SELECT sum(cityHash64(*)) FROM hits_experimental; +DROP TABLE hits_experimental; + +CREATE TABLE hits_experimental (Title String CODEC(Density(3))) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO hits_experimental SELECT Title FROM test.hits; +SELECT sum(cityHash64(*)) FROM hits_experimental; +DROP TABLE hits_experimental; + +-- LZSSE + +CREATE TABLE hits_experimental (Title String CODEC(LZSSE2)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO hits_experimental SELECT Title FROM test.hits ORDER BY Title LIMIT 100000; +SELECT sum(cityHash64(*)) FROM hits_experimental; +DROP TABLE hits_experimental; + +CREATE TABLE hits_experimental (Title String CODEC(LZSSE4)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO hits_experimental SELECT Title FROM test.hits ORDER BY Title LIMIT 100000; +SELECT sum(cityHash64(*)) FROM hits_experimental; +DROP TABLE hits_experimental; + +CREATE TABLE hits_experimental (Title String CODEC(LZSSE8)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO hits_experimental SELECT Title FROM test.hits ORDER BY Title LIMIT 100000; +SELECT sum(cityHash64(*)) FROM hits_experimental; +DROP TABLE hits_experimental; From 0ab67f74915421b5e72c8ed1c8614dcf9ebe741f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 May 2021 06:57:02 +0300 Subject: [PATCH 41/44] Fix YAML lint --- programs/server/config-example.yaml | 101 ++++++++++++++-------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/programs/server/config-example.yaml b/programs/server/config-example.yaml index af62455ccef..3e275adf2fa 100644 --- a/programs/server/config-example.yaml +++ b/programs/server/config-example.yaml @@ -391,56 +391,57 @@ user_directories: local_directory: # Path to folder where users created by SQL commands are stored. path: /var/lib/clickhouse/access/ - # To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section - # with the following parameters: - # server - one of LDAP server names defined in 'ldap_servers' config section above. - # This parameter is mandatory and cannot be empty. - # roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. - # If no roles are specified here or assigned during role mapping (below), user will not be able to perform any - # actions after authentication. - # role_mapping - section with LDAP search parameters and mapping rules. - # When a user authenticates, while still bound to LDAP, an LDAP search is performed using search_filter and the - # name of the logged in user. For each entry found during that search, the value of the specified attribute is - # extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the - # value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by - # CREATE ROLE command. - # There can be multiple 'role_mapping' sections defined inside the same 'ldap' section. All of them will be - # applied. - # base_dn - template used to construct the base DN for the LDAP search. - # The resulting DN will be constructed by replacing all '{user_name}', '{bind_dn}', and '{user_dn}' - # substrings of the template with the actual user name, bind DN, and user DN during each LDAP search. - # scope - scope of the LDAP search. - # Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default). - # search_filter - template used to construct the search filter for the LDAP search. - # The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', '{user_dn}', and - # '{base_dn}' substrings of the template with the actual user name, bind DN, user DN, and base DN during - # each LDAP search. - # Note, that the special characters must be escaped properly in XML. - # attribute - attribute name whose values will be returned by the LDAP search. 'cn', by default. - # prefix - prefix, that will be expected to be in front of each string in the original list of strings returned by - # the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated - # as local role names. Empty, by default. - # Example: - # ldap: - # server: my_ldap_server - # roles: - # my_local_role1: '' - # my_local_role2: '' - # role_mapping: - # base_dn: 'ou=groups,dc=example,dc=com' - # scope: subtree - # search_filter: '(&(objectClass=groupOfNames)(member={bind_dn}))' - # attribute: cn - # prefix: clickhouse_ - # Example (typical Active Directory with role mapping that relies on the detected user DN): - # ldap: - # server: my_ad_server - # role_mapping: - # base_dn: 'CN=Users,DC=example,DC=com' - # attribute: CN - # scope: subtree - # search_filter: '(&(objectClass=group)(member={user_dn}))' - # prefix: clickhouse_ + +# # To add an LDAP server as a remote user directory of users that are not defined locally, define a single 'ldap' section +# # with the following parameters: +# # server - one of LDAP server names defined in 'ldap_servers' config section above. +# # This parameter is mandatory and cannot be empty. +# # roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. +# # If no roles are specified here or assigned during role mapping (below), user will not be able to perform any +# # actions after authentication. +# # role_mapping - section with LDAP search parameters and mapping rules. +# # When a user authenticates, while still bound to LDAP, an LDAP search is performed using search_filter and the +# # name of the logged in user. For each entry found during that search, the value of the specified attribute is +# # extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the +# # value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by +# # CREATE ROLE command. +# # There can be multiple 'role_mapping' sections defined inside the same 'ldap' section. All of them will be +# # applied. +# # base_dn - template used to construct the base DN for the LDAP search. +# # The resulting DN will be constructed by replacing all '{user_name}', '{bind_dn}', and '{user_dn}' +# # substrings of the template with the actual user name, bind DN, and user DN during each LDAP search. +# # scope - scope of the LDAP search. +# # Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default). +# # search_filter - template used to construct the search filter for the LDAP search. +# # The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', '{user_dn}', and +# # '{base_dn}' substrings of the template with the actual user name, bind DN, user DN, and base DN during +# # each LDAP search. +# # Note, that the special characters must be escaped properly in XML. +# # attribute - attribute name whose values will be returned by the LDAP search. 'cn', by default. +# # prefix - prefix, that will be expected to be in front of each string in the original list of strings returned by +# # the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated +# # as local role names. Empty, by default. +# # Example: +# # ldap: +# # server: my_ldap_server +# # roles: +# # my_local_role1: '' +# # my_local_role2: '' +# # role_mapping: +# # base_dn: 'ou=groups,dc=example,dc=com' +# # scope: subtree +# # search_filter: '(&(objectClass=groupOfNames)(member={bind_dn}))' +# # attribute: cn +# # prefix: clickhouse_ +# # Example (typical Active Directory with role mapping that relies on the detected user DN): +# # ldap: +# # server: my_ad_server +# # role_mapping: +# # base_dn: 'CN=Users,DC=example,DC=com' +# # attribute: CN +# # scope: subtree +# # search_filter: '(&(objectClass=group)(member={user_dn}))' +# # prefix: clickhouse_ # Default profile of settings. default_profile: default From 374923ab90bdba0d107427fb1e38f96f94d22a62 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 May 2021 03:48:11 +0300 Subject: [PATCH 42/44] Update integration test --- .../configs/allow_experimental_codecs.xml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/integration/test_non_default_compression/configs/allow_experimental_codecs.xml diff --git a/tests/integration/test_non_default_compression/configs/allow_experimental_codecs.xml b/tests/integration/test_non_default_compression/configs/allow_experimental_codecs.xml new file mode 100644 index 00000000000..2c7de77cd37 --- /dev/null +++ b/tests/integration/test_non_default_compression/configs/allow_experimental_codecs.xml @@ -0,0 +1,7 @@ + + + + 1 + + + From 31f02b37805e5063c36c55930cd3805a13b8c853 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 6 Jun 2021 02:21:42 +0300 Subject: [PATCH 43/44] Remove unused code --- programs/compressor/Compressor.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index 35534eaee9e..d47372631fe 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -20,9 +19,6 @@ #include #include -#include -#include - namespace DB { @@ -81,7 +77,6 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) ("block-size,b", po::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") ("hc", "use LZ4HC instead of LZ4") ("zstd", "use ZSTD instead of LZ4") - ("param", po::value(), "extra params for compression algorithm") ("codec", po::value>()->multitoken(), "use codecs combination instead of LZ4") ("level", po::value(), "compression level for codecs specified via flags") ("none", "use no compression instead of LZ4") From 8d851457254cf7fe90edde613cffeeeb011495eb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 6 Jun 2021 02:26:52 +0300 Subject: [PATCH 44/44] Remove experimental compression codecs --- .gitmodules | 9 - CMakeLists.txt | 3 - cmake/find/density.cmake | 9 - cmake/find/lizard.cmake | 9 - cmake/find/lzsse.cmake | 14 -- contrib/CMakeLists.txt | 12 -- contrib/density | 1 - contrib/density-cmake/CMakeLists.txt | 47 ----- contrib/lizard | 1 - contrib/lizard-cmake/CMakeLists.txt | 19 -- contrib/lzsse | 1 - contrib/lzsse-cmake/CMakeLists.txt | 21 -- src/CMakeLists.txt | 12 -- src/Common/config.h.in | 3 - src/Compression/CompressionCodecDensity.cpp | 122 ------------ src/Compression/CompressionCodecLZSSE.cpp | 180 ------------------ src/Compression/CompressionCodecLizard.cpp | 121 ------------ src/Compression/CompressionFactory.cpp | 21 -- src/Compression/CompressionInfo.h | 6 - src/Core/Settings.h | 2 +- src/Core/config_core.h.in | 3 - .../test_non_default_compression/test.py | 16 -- .../00096_experimental_codecs.reference | 8 - .../1_stateful/00096_experimental_codecs.sql | 61 ------ 24 files changed, 1 insertion(+), 700 deletions(-) delete mode 100644 cmake/find/density.cmake delete mode 100644 cmake/find/lizard.cmake delete mode 100644 cmake/find/lzsse.cmake delete mode 160000 contrib/density delete mode 100644 contrib/density-cmake/CMakeLists.txt delete mode 160000 contrib/lizard delete mode 100644 contrib/lizard-cmake/CMakeLists.txt delete mode 160000 contrib/lzsse delete mode 100644 contrib/lzsse-cmake/CMakeLists.txt delete mode 100644 src/Compression/CompressionCodecDensity.cpp delete mode 100644 src/Compression/CompressionCodecLZSSE.cpp delete mode 100644 src/Compression/CompressionCodecLizard.cpp delete mode 100644 tests/queries/1_stateful/00096_experimental_codecs.reference delete mode 100644 tests/queries/1_stateful/00096_experimental_codecs.sql diff --git a/.gitmodules b/.gitmodules index e59fbd2ca7e..ab7c8a7c94d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -228,15 +228,6 @@ [submodule "contrib/datasketches-cpp"] path = contrib/datasketches-cpp url = https://github.com/ClickHouse-Extras/datasketches-cpp.git -[submodule "contrib/lizard"] - path = contrib/lizard - url = https://github.com/inikep/lizard -[submodule "contrib/density"] - path = contrib/density - url = https://github.com/centaurean/density.git -[submodule "contrib/lzsse"] - path = contrib/lzsse - url = https://github.com/ConorStokes/LZSSE.git [submodule "contrib/yaml-cpp"] path = contrib/yaml-cpp url = https://github.com/ClickHouse-Extras/yaml-cpp.git diff --git a/CMakeLists.txt b/CMakeLists.txt index fd98ef95b67..ce0f58e2521 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -527,9 +527,6 @@ include (cmake/find/rocksdb.cmake) include (cmake/find/libpqxx.cmake) include (cmake/find/nuraft.cmake) include (cmake/find/yaml-cpp.cmake) -include (cmake/find/lizard.cmake) -include (cmake/find/lzsse.cmake) -include (cmake/find/density.cmake) if(NOT USE_INTERNAL_PARQUET_LIBRARY) set (ENABLE_ORC OFF CACHE INTERNAL "") diff --git a/cmake/find/density.cmake b/cmake/find/density.cmake deleted file mode 100644 index f4963d67709..00000000000 --- a/cmake/find/density.cmake +++ /dev/null @@ -1,9 +0,0 @@ -option(USE_DENSITY "Enable DENSITY experimental compression library" ${ENABLE_LIBRARIES}) - -if (NOT USE_DENSITY) - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/density/README.md") - message (ERROR "submodule contrib/density is missing. to fix try run: \n git submodule update --init --recursive") -endif() diff --git a/cmake/find/lizard.cmake b/cmake/find/lizard.cmake deleted file mode 100644 index 27c8945c45a..00000000000 --- a/cmake/find/lizard.cmake +++ /dev/null @@ -1,9 +0,0 @@ -option(USE_LIZARD "Enable Lizard experimental compression library" ${ENABLE_LIBRARIES}) - -if (NOT USE_LIZARD) - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/lizard/README.md") - message (ERROR "submodule contrib/lizard is missing. to fix try run: \n git submodule update --init --recursive") -endif() diff --git a/cmake/find/lzsse.cmake b/cmake/find/lzsse.cmake deleted file mode 100644 index 763483d8ea9..00000000000 --- a/cmake/find/lzsse.cmake +++ /dev/null @@ -1,14 +0,0 @@ -set (DEFAULT_USE_LZSSE 0) -if (ENABLE_LIBRARIES AND ARCH_AMD64) - set (DEFAULT_USE_LZSSE 1) -endif() - -option(USE_LZSSE "Enable LZSSE experimental compression library" ${DEFAULT_USE_LZSSE}) - -if (NOT USE_LZSSE) - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/lzsse/README.md") - message (ERROR "submodule contrib/lzsse is missing. to fix try run: \n git submodule update --init --recursive") -endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index eccd1dc1430..710c8c7fca5 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -58,18 +58,6 @@ if (USE_INTERNAL_XZ_LIBRARY) add_subdirectory (xz) endif() -if (USE_LIZARD) - add_subdirectory (lizard-cmake) -endif () - -if (USE_DENSITY) - add_subdirectory (density-cmake) -endif () - -if (USE_LZSSE) - add_subdirectory (lzsse-cmake) -endif () - add_subdirectory (poco-cmake) add_subdirectory (croaring-cmake) diff --git a/contrib/density b/contrib/density deleted file mode 160000 index 67bd584bd74..00000000000 --- a/contrib/density +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 67bd584bd7414d4bc1418e5b7fcf9d68e44c3f28 diff --git a/contrib/density-cmake/CMakeLists.txt b/contrib/density-cmake/CMakeLists.txt deleted file mode 100644 index 844459c4105..00000000000 --- a/contrib/density-cmake/CMakeLists.txt +++ /dev/null @@ -1,47 +0,0 @@ -SET (USE_INTERNAL_DENSITY_LIBRARY 1) -SET (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/density") - -# cd contrib/density -# find . -name '*.c' | grep -vP 'deprecated|legacy' | sort | sed 's/^\./ ${LIBRARY_DIR}/' -SET (Sources - ${LIBRARY_DIR}/src/algorithms/algorithms.c - ${LIBRARY_DIR}/src/algorithms/chameleon/core/chameleon_decode.c - ${LIBRARY_DIR}/src/algorithms/chameleon/core/chameleon_encode.c - ${LIBRARY_DIR}/src/algorithms/cheetah/core/cheetah_decode.c - ${LIBRARY_DIR}/src/algorithms/cheetah/core/cheetah_encode.c - ${LIBRARY_DIR}/src/algorithms/dictionaries.c - ${LIBRARY_DIR}/src/algorithms/lion/core/lion_decode.c - ${LIBRARY_DIR}/src/algorithms/lion/core/lion_encode.c - ${LIBRARY_DIR}/src/algorithms/lion/forms/lion_form_model.c - ${LIBRARY_DIR}/src/buffers/buffer.c - ${LIBRARY_DIR}/src/globals.c - ${LIBRARY_DIR}/src/structure/header.c -) - -# cd contrib/density -# find . -name '*.h' | grep -vP 'deprecated|legacy' | sort | sed 's/^\./ ${LIBRARY_DIR}/' -SET (Headers - ${LIBRARY_DIR}/src/algorithms/algorithms.h - ${LIBRARY_DIR}/src/algorithms/chameleon/chameleon.h - ${LIBRARY_DIR}/src/algorithms/chameleon/core/chameleon_decode.h - ${LIBRARY_DIR}/src/algorithms/chameleon/core/chameleon_encode.h - ${LIBRARY_DIR}/src/algorithms/chameleon/dictionary/chameleon_dictionary.h - ${LIBRARY_DIR}/src/algorithms/cheetah/cheetah.h - ${LIBRARY_DIR}/src/algorithms/cheetah/core/cheetah_decode.h - ${LIBRARY_DIR}/src/algorithms/cheetah/core/cheetah_encode.h - ${LIBRARY_DIR}/src/algorithms/cheetah/dictionary/cheetah_dictionary.h - ${LIBRARY_DIR}/src/algorithms/dictionaries.h - ${LIBRARY_DIR}/src/algorithms/lion/core/lion_decode.h - ${LIBRARY_DIR}/src/algorithms/lion/core/lion_encode.h - ${LIBRARY_DIR}/src/algorithms/lion/dictionary/lion_dictionary.h - ${LIBRARY_DIR}/src/algorithms/lion/forms/lion_form_model.h - ${LIBRARY_DIR}/src/algorithms/lion/lion.h - ${LIBRARY_DIR}/src/buffers/buffer.h - ${LIBRARY_DIR}/src/density_api.h - ${LIBRARY_DIR}/src/globals.h - ${LIBRARY_DIR}/src/structure/header.h -) - -ADD_LIBRARY(density ${Sources} ${Headers}) - -target_include_directories (density PUBLIC ${LIBRARY_DIR}) diff --git a/contrib/lizard b/contrib/lizard deleted file mode 160000 index af8518ccb8c..00000000000 --- a/contrib/lizard +++ /dev/null @@ -1 +0,0 @@ -Subproject commit af8518ccb8c68e062a8c80205ff07d56a2e77dd4 diff --git a/contrib/lizard-cmake/CMakeLists.txt b/contrib/lizard-cmake/CMakeLists.txt deleted file mode 100644 index 7fe5274ecb6..00000000000 --- a/contrib/lizard-cmake/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -SET (USE_INTERNAL_LIZARD_LIBRARY 1) -SET (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/lizard") - -SET (Sources - "${LIBRARY_DIR}/lib/lizard_decompress.c" - "${LIBRARY_DIR}/lib/lizard_compress.c" -) - -SET (Headers - "${LIBRARY_DIR}/lib/lizard_compress.h" - "${LIBRARY_DIR}/lib/lizard_common.h" -) - -ADD_LIBRARY(lizard ${Sources} ${Headers}) - -target_include_directories (lizard PUBLIC ${LIBRARY_DIR}) - -# It is using some symbols (HUF_* and FSE_*) from zstd. -target_link_libraries (lizard PRIVATE zstd) diff --git a/contrib/lzsse b/contrib/lzsse deleted file mode 160000 index 1847c3e8279..00000000000 --- a/contrib/lzsse +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1847c3e82794400deb56edd30d8aa3f445fd000b diff --git a/contrib/lzsse-cmake/CMakeLists.txt b/contrib/lzsse-cmake/CMakeLists.txt deleted file mode 100644 index 5ec5b312009..00000000000 --- a/contrib/lzsse-cmake/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -SET (USE_INTERNAL_LZSSE_LIBRARY 1) -SET (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/lzsse") - -SET (Sources - "${LIBRARY_DIR}/lzsse2/lzsse2.cpp" - "${LIBRARY_DIR}/lzsse4/lzsse4.cpp" - "${LIBRARY_DIR}/lzsse8/lzsse8.cpp" -) - -SET (Headers - "${LIBRARY_DIR}/lzsse2/lzsse2.h" - "${LIBRARY_DIR}/lzsse4/lzsse4.h" - "${LIBRARY_DIR}/lzsse8/lzsse8.h" - "${LIBRARY_DIR}/lzsse2/lzsse2_platform.h" - "${LIBRARY_DIR}/lzsse4/lzsse4_platform.h" - "${LIBRARY_DIR}/lzsse8/lzsse8_platform.h" -) - -ADD_LIBRARY(lzsse ${Sources} ${Headers}) - -target_include_directories (lzsse SYSTEM PUBLIC ${LIBRARY_DIR}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index af5ea3772ae..26a68fcbd14 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -380,18 +380,6 @@ if (XZ_LIBRARY) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${XZ_INCLUDE_DIR}) endif() -if (USE_LIZARD) - dbms_target_link_libraries(PRIVATE lizard) -endif() - -if (USE_DENSITY) - dbms_target_link_libraries(PRIVATE density) -endif() - -if (USE_LZSSE) - dbms_target_link_libraries(PRIVATE lzsse) -endif() - if (USE_ICU) dbms_target_link_libraries (PRIVATE ${ICU_LIBRARIES}) dbms_target_include_directories (SYSTEM PRIVATE ${ICU_INCLUDE_DIRS}) diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 7396e6f6222..df27a7b7d9e 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -16,7 +16,4 @@ #cmakedefine01 USE_STATS #cmakedefine01 USE_DATASKETCHES #cmakedefine01 USE_YAML_CPP -#cmakedefine01 USE_LIZARD -#cmakedefine01 USE_DENSITY -#cmakedefine01 USE_LZSSE #cmakedefine01 CLICKHOUSE_SPLIT_BINARY diff --git a/src/Compression/CompressionCodecDensity.cpp b/src/Compression/CompressionCodecDensity.cpp deleted file mode 100644 index e462ff2765b..00000000000 --- a/src/Compression/CompressionCodecDensity.cpp +++ /dev/null @@ -1,122 +0,0 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif - -#if USE_DENSITY - -#include -#include -#include -#include - -#include - - -namespace DB -{ - -class CompressionCodecDensity : public ICompressionCodec -{ -public: - explicit CompressionCodecDensity(DENSITY_ALGORITHM algo_); - - uint8_t getMethodByte() const override; - - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - - void updateHash(SipHash & hash) const override; - -protected: - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - - bool isCompression() const override { return true; } - bool isGenericCompression() const override { return true; } - bool isExperimental() const override { return true; } - -private: - const DENSITY_ALGORITHM algo; -}; - - -namespace ErrorCodes -{ - extern const int CANNOT_COMPRESS; - extern const int CANNOT_DECOMPRESS; - extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; - extern const int ILLEGAL_CODEC_PARAMETER; -} - -CompressionCodecDensity::CompressionCodecDensity(DENSITY_ALGORITHM algo_) : algo(algo_) -{ - setCodecDescription("Density", {std::make_shared(static_cast(algo))}); -} - -uint8_t CompressionCodecDensity::getMethodByte() const -{ - return static_cast(CompressionMethodByte::Density); -} - -void CompressionCodecDensity::updateHash(SipHash & hash) const -{ - getCodecDesc()->updateTreeHash(hash); -} - -UInt32 CompressionCodecDensity::getMaxCompressedDataSize(UInt32 uncompressed_size) const -{ - return density_compress_safe_size(uncompressed_size); -} - -UInt32 CompressionCodecDensity::doCompressData(const char * source, UInt32 source_size, char * dest) const -{ - density_processing_result res = density_compress(reinterpret_cast(source), source_size, reinterpret_cast(dest), density_compress_safe_size(source_size), algo); - if (res.state != DENSITY_STATE_OK) - throw Exception("Cannot compress block with Density", ErrorCodes::CANNOT_COMPRESS); - return res.bytesWritten; -} - -void CompressionCodecDensity::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const -{ - density_processing_result res = density_decompress(reinterpret_cast(source), source_size, reinterpret_cast(dest), density_decompress_safe_size(uncompressed_size)); - if (res.state != DENSITY_STATE_OK) - throw Exception("Cannot decompress block with Density", ErrorCodes::CANNOT_DECOMPRESS); -} - -void registerCodecDensity(CompressionCodecFactory & factory) -{ - UInt8 method_code = UInt8(CompressionMethodByte::Density); - factory.registerCompressionCodec( - "Density", - method_code, - [&](const ASTPtr & arguments) -> CompressionCodecPtr - { - DENSITY_ALGORITHM algorithm = DENSITY_ALGORITHM_CHAMELEON; - - if (arguments && !arguments->children.empty()) - { - if (arguments->children.size() != 1) - throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, - "Density codec must have only one parameter, given {}", arguments->children.size()); - - const auto children = arguments->children; - - const auto * algo_literal = children[0]->as(); - if (!algo_literal || algo_literal->value.getType() != Field::Types::UInt64) - throw Exception("Density codec argument must be integer", - ErrorCodes::ILLEGAL_CODEC_PARAMETER); - - const UInt64 algorithm_num = algo_literal->value.safeGet(); - if (algorithm_num < 1 || algorithm_num > 3) - throw Exception("Density codec level can be 1, 2 or 3.", ErrorCodes::ILLEGAL_CODEC_PARAMETER); - - algorithm = static_cast(algorithm_num); - } - - return std::make_shared(algorithm); - }); -} - -} - -#endif diff --git a/src/Compression/CompressionCodecLZSSE.cpp b/src/Compression/CompressionCodecLZSSE.cpp deleted file mode 100644 index 62fb641bc12..00000000000 --- a/src/Compression/CompressionCodecLZSSE.cpp +++ /dev/null @@ -1,180 +0,0 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif - -#if USE_LZSSE - -#include -#include -#include -#include - -#include -#include -#include - - -namespace DB -{ - -class CompressionCodecLZSSE : public ICompressionCodec -{ -public: - explicit CompressionCodecLZSSE(UInt32 type_, UInt32 level_); - - uint8_t getMethodByte() const override; - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - void updateHash(SipHash & hash) const override; - -protected: - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - bool isCompression() const override { return true; } - bool isGenericCompression() const override { return true; } - bool isExperimental() const override { return true; } - -private: - const UInt32 type; - const UInt32 level; -}; - - -namespace ErrorCodes -{ - extern const int CANNOT_COMPRESS; - extern const int CANNOT_DECOMPRESS; - extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; - extern const int ILLEGAL_CODEC_PARAMETER; - extern const int LOGICAL_ERROR; -} - -CompressionCodecLZSSE::CompressionCodecLZSSE(UInt32 type_, UInt32 level_) : type(type_), level(level_) -{ - if (type != 2 && type != 4 && type != 8) - throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no LZSSE{} codec", type); - - setCodecDescription(fmt::format("LZSSE{}", type), {std::make_shared(static_cast(level))}); -} - -uint8_t CompressionCodecLZSSE::getMethodByte() const -{ - switch (type) - { - case 2: return static_cast(CompressionMethodByte::LZSSE2); - case 4: return static_cast(CompressionMethodByte::LZSSE4); - case 8: return static_cast(CompressionMethodByte::LZSSE8); - default: - throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no LZSSE{} codec", type); - } -} - -void CompressionCodecLZSSE::updateHash(SipHash & hash) const -{ - getCodecDesc()->updateTreeHash(hash); -} - -UInt32 CompressionCodecLZSSE::getMaxCompressedDataSize(UInt32 uncompressed_size) const -{ - return uncompressed_size; -} - -UInt32 CompressionCodecLZSSE::doCompressData(const char * source, UInt32 source_size, char * dest) const -{ - UInt32 res = 0; - switch (type) - { - case 2: - { - LZSSE2_OptimalParseState * state = LZSSE2_MakeOptimalParseState(source_size); - res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level); - LZSSE2_FreeOptimalParseState(state); - break; - } - case 4: - { - LZSSE4_OptimalParseState * state = LZSSE4_MakeOptimalParseState(source_size); - res = LZSSE4_CompressOptimalParse(state, source, source_size, dest, source_size, level); - LZSSE4_FreeOptimalParseState(state); - break; - } - case 8: - { - LZSSE8_OptimalParseState * state = LZSSE8_MakeOptimalParseState(source_size); - res = LZSSE8_CompressOptimalParse(state, source, source_size, dest, source_size, level); - LZSSE8_FreeOptimalParseState(state); - break; - } - default: - break; - } - - if (res == 0) - throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress block with LZSSE{}", type); - return res; -} - -void CompressionCodecLZSSE::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const -{ - UInt32 res = 0; - switch (type) - { - case 2: - { - res = LZSSE2_Decompress(source, source_size, dest, uncompressed_size); - break; - } - case 4: - { - res = LZSSE4_Decompress(source, source_size, dest, uncompressed_size); - break; - } - case 8: - { - res = LZSSE8_Decompress(source, source_size, dest, uncompressed_size); - break; - } - default: - break; - } - if (res < uncompressed_size) - throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress block with LZSSE{}", type); -} - -void registerCodecsLZSSE(CompressionCodecFactory & factory) -{ - for (auto [type, method_byte] : std::initializer_list> - { - {2, CompressionMethodByte::LZSSE2}, - {4, CompressionMethodByte::LZSSE4}, - {8, CompressionMethodByte::LZSSE8} - }) - { - factory.registerCompressionCodec( - fmt::format("LZSSE{}", type), - uint8_t(method_byte), - [type = type](const ASTPtr & arguments) -> CompressionCodecPtr - { - int level = 1; - if (arguments && !arguments->children.empty()) - { - if (arguments->children.size() != 1) - throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, - "LZSSE{} codec must have 1 parameter, {} given", type, arguments->children.size()); - - const auto children = arguments->children; - const auto * level_literal = children[0]->as(); - if (!level_literal) - throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, - "LZSSE{} first codec argument must be integer", type); - - level = level_literal->value.safeGet(); - } - - return std::make_shared(type, level); - }); - } -} - -} - -#endif diff --git a/src/Compression/CompressionCodecLizard.cpp b/src/Compression/CompressionCodecLizard.cpp deleted file mode 100644 index cd2d49a0538..00000000000 --- a/src/Compression/CompressionCodecLizard.cpp +++ /dev/null @@ -1,121 +0,0 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif - -#if USE_LIZARD - -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -class CompressionCodecLizard : public ICompressionCodec -{ -public: - static constexpr auto LIZARD_DEFAULT_LEVEL = 1; - - explicit CompressionCodecLizard(int level_); - - uint8_t getMethodByte() const override; - - UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override; - - void updateHash(SipHash & hash) const override; - -protected: - UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; - void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; - - bool isCompression() const override { return true; } - bool isGenericCompression() const override { return true; } - bool isExperimental() const override { return true; } - -private: - const int level; -}; - - -namespace ErrorCodes -{ - extern const int CANNOT_COMPRESS; - extern const int CANNOT_DECOMPRESS; - extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; - extern const int ILLEGAL_CODEC_PARAMETER; -} - -CompressionCodecLizard::CompressionCodecLizard(int level_) : level(level_) -{ - setCodecDescription("Lizard", {std::make_shared(static_cast(level))}); -} - -uint8_t CompressionCodecLizard::getMethodByte() const -{ - return static_cast(CompressionMethodByte::Lizard); -} - -void CompressionCodecLizard::updateHash(SipHash & hash) const -{ - getCodecDesc()->updateTreeHash(hash); -} - -UInt32 CompressionCodecLizard::getMaxCompressedDataSize(UInt32 uncompressed_size) const -{ - return Lizard_compressBound(uncompressed_size); -} - -UInt32 CompressionCodecLizard::doCompressData(const char * source, UInt32 source_size, char * dest) const -{ - int res = Lizard_compress(source, dest, source_size, Lizard_compressBound(source_size), level); - - if (res == 0) - throw Exception("Cannot compress block with Lizard", ErrorCodes::CANNOT_COMPRESS); - return res; -} - -void CompressionCodecLizard::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const -{ - int res = Lizard_decompress_safe(source, dest, source_size, uncompressed_size); - - if (res < 0) - throw Exception("Cannot compress block with Lizard", ErrorCodes::CANNOT_DECOMPRESS); -} - -void registerCodecLizard(CompressionCodecFactory & factory) -{ - UInt8 method_code = UInt8(CompressionMethodByte::Lizard); - factory.registerCompressionCodec( - "Lizard", - method_code, - [&](const ASTPtr & arguments) -> CompressionCodecPtr - { - int level = CompressionCodecLizard::LIZARD_DEFAULT_LEVEL; - if (arguments && !arguments->children.empty()) - { - if (arguments->children.size() > 1) - throw Exception( - "Lizard codec must have 1 parameter, given " + std::to_string(arguments->children.size()), - ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE); - - const auto children = arguments->children; - const auto * literal = children[0]->as(); - if (!literal) - throw Exception("Lizard codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER); - - level = literal->value.safeGet(); - // compression level will be truncated to LIZARD_MAX_CLEVEL if it is greater and to LIZARD_MIN_CLEVEL if it is less - //if (level > 1)//ZSTD_maxCLevel()) - // throw Exception("Lizard codec can't have level more that " + toString(1/*ZSTD_maxCLevel()*/) + ", given " + toString(level), ErrorCodes::ILLEGAL_CODEC_PARAMETER); - } - - return std::make_shared(level); - }); -} - -} - -#endif diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index fbd369fdfe7..dc65713471c 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -339,18 +339,6 @@ void registerCodecDoubleDelta(CompressionCodecFactory & factory); void registerCodecGorilla(CompressionCodecFactory & factory); void registerCodecMultiple(CompressionCodecFactory & factory); -#if USE_LIZARD -void registerCodecLizard(CompressionCodecFactory & factory); -#endif - -#if USE_DENSITY -void registerCodecDensity(CompressionCodecFactory & factory); -#endif - -#if USE_LZSSE -void registerCodecsLZSSE(CompressionCodecFactory & factory); -#endif - CompressionCodecFactory::CompressionCodecFactory() { registerCodecLZ4(*this); @@ -362,15 +350,6 @@ CompressionCodecFactory::CompressionCodecFactory() registerCodecDoubleDelta(*this); registerCodecGorilla(*this); registerCodecMultiple(*this); -#if USE_LIZARD - registerCodecLizard(*this); -#endif -#if USE_DENSITY - registerCodecDensity(*this); -#endif -#if USE_LZSSE - registerCodecsLZSSE(*this); -#endif default_codec = get("LZ4", {}); } diff --git a/src/Compression/CompressionInfo.h b/src/Compression/CompressionInfo.h index 44bed7d3c1d..58a39bb12a4 100644 --- a/src/Compression/CompressionInfo.h +++ b/src/Compression/CompressionInfo.h @@ -43,12 +43,6 @@ enum class CompressionMethodByte : uint8_t T64 = 0x93, DoubleDelta = 0x94, Gorilla = 0x95, - Lizard = 0x96, - Density = 0x97, - LZSSE2 = 0x98, - LZSSE4 = 0x99, - LZSSE8 = 0xA0, - }; } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index f4692b7e41f..08b69e52bbc 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -240,7 +240,7 @@ class IColumn; M(Bool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \ M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \ - M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (LZSSE2/4/8, Lizard, Density). These codecs are provided for evaluation purposes.", 0) \ + M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \ M(UInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \ M(UInt64, query_profiler_real_time_period_ns, 1000000000, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(UInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ diff --git a/src/Core/config_core.h.in b/src/Core/config_core.h.in index da923ca23a1..e250e013913 100644 --- a/src/Core/config_core.h.in +++ b/src/Core/config_core.h.in @@ -15,6 +15,3 @@ #cmakedefine01 USE_LIBPQXX #cmakedefine01 USE_NURAFT #cmakedefine01 USE_KRB5 -#cmakedefine01 USE_LIZARD -#cmakedefine01 USE_DENSITY -#cmakedefine01 USE_LZSSE diff --git a/tests/integration/test_non_default_compression/test.py b/tests/integration/test_non_default_compression/test.py index 54e14132d3b..0cfffd28e12 100644 --- a/tests/integration/test_non_default_compression/test.py +++ b/tests/integration/test_non_default_compression/test.py @@ -138,19 +138,3 @@ def test_uncompressed_cache_plus_zstd_codec(start_cluster): assert node5.query( "SELECT max(length(data)) from compression_codec_multiple_with_key GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" - -def test_experimental_codecs(start_cluster): - node6.query(""" - CREATE TABLE compression_experimental_codecs ( - somedate Date CODEC(Lizard(12)), - id UInt64 CODEC(Density(3)), - data String CODEC(LZSSE4(3)) - ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; - """) - - node6.query( - "INSERT INTO compression_experimental_codecs VALUES(toDate('2018-10-12'), 100000, '{}')".format( - 'c' * 10000)) - - assert node6.query( - "SELECT max(length(data)) from compression_experimental_codecs GROUP BY data ORDER BY max(length(data)) DESC LIMIT 1") == "10000\n" diff --git a/tests/queries/1_stateful/00096_experimental_codecs.reference b/tests/queries/1_stateful/00096_experimental_codecs.reference deleted file mode 100644 index a34182c23e5..00000000000 --- a/tests/queries/1_stateful/00096_experimental_codecs.reference +++ /dev/null @@ -1,8 +0,0 @@ -17415138241754778329 -17415138241754778329 -17415138241754778329 -17415138241754778329 -17415138241754778329 -3798944011853532000 -3798944011853532000 -3798944011853532000 diff --git a/tests/queries/1_stateful/00096_experimental_codecs.sql b/tests/queries/1_stateful/00096_experimental_codecs.sql deleted file mode 100644 index e085e55f886..00000000000 --- a/tests/queries/1_stateful/00096_experimental_codecs.sql +++ /dev/null @@ -1,61 +0,0 @@ -DROP TABLE IF EXISTS hits_experimental; - --- It's not allowed to create a table with experimental codecs unless the user turns off the safety switch. -CREATE TABLE hits_experimental (Title String CODEC(Lizard(10))) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 36 } - -SET allow_experimental_codecs = 1; - --- Lizard - -CREATE TABLE hits_experimental (Title String CODEC(Lizard(10))) ENGINE = MergeTree ORDER BY tuple(); -INSERT INTO hits_experimental SELECT Title FROM test.hits; -SELECT sum(cityHash64(*)) FROM hits_experimental; - --- It's always allowed to attach a table with experimental codecs. -DETACH TABLE hits_experimental; -SET allow_experimental_codecs = 0; -ATTACH TABLE hits_experimental; -SELECT sum(cityHash64(*)) FROM hits_experimental; -SET allow_experimental_codecs = 1; - -DROP TABLE hits_experimental; - --- Density - --- Check out of range levels of Density. -CREATE TABLE hits_experimental (Title String CODEC(Density(-1))) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 433 } -CREATE TABLE hits_experimental (Title String CODEC(Density(0))) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 433 } -CREATE TABLE hits_experimental (Title String CODEC(Density(4))) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 433 } -CREATE TABLE hits_experimental (Title String CODEC(Density('hello'))) ENGINE = MergeTree ORDER BY tuple(); -- { serverError 433 } - -CREATE TABLE hits_experimental (Title String CODEC(Density(1))) ENGINE = MergeTree ORDER BY tuple(); -INSERT INTO hits_experimental SELECT Title FROM test.hits; -SELECT sum(cityHash64(*)) FROM hits_experimental; -DROP TABLE hits_experimental; - -CREATE TABLE hits_experimental (Title String CODEC(Density(2))) ENGINE = MergeTree ORDER BY tuple(); -INSERT INTO hits_experimental SELECT Title FROM test.hits; -SELECT sum(cityHash64(*)) FROM hits_experimental; -DROP TABLE hits_experimental; - -CREATE TABLE hits_experimental (Title String CODEC(Density(3))) ENGINE = MergeTree ORDER BY tuple(); -INSERT INTO hits_experimental SELECT Title FROM test.hits; -SELECT sum(cityHash64(*)) FROM hits_experimental; -DROP TABLE hits_experimental; - --- LZSSE - -CREATE TABLE hits_experimental (Title String CODEC(LZSSE2)) ENGINE = MergeTree ORDER BY tuple(); -INSERT INTO hits_experimental SELECT Title FROM test.hits ORDER BY Title LIMIT 100000; -SELECT sum(cityHash64(*)) FROM hits_experimental; -DROP TABLE hits_experimental; - -CREATE TABLE hits_experimental (Title String CODEC(LZSSE4)) ENGINE = MergeTree ORDER BY tuple(); -INSERT INTO hits_experimental SELECT Title FROM test.hits ORDER BY Title LIMIT 100000; -SELECT sum(cityHash64(*)) FROM hits_experimental; -DROP TABLE hits_experimental; - -CREATE TABLE hits_experimental (Title String CODEC(LZSSE8)) ENGINE = MergeTree ORDER BY tuple(); -INSERT INTO hits_experimental SELECT Title FROM test.hits ORDER BY Title LIMIT 100000; -SELECT sum(cityHash64(*)) FROM hits_experimental; -DROP TABLE hits_experimental;