thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 07:31:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

add contribs

Browse Source
This commit is contained in:
fibersel 2021-05-06 15:55:03 +03:00
parent 9f7733567b
commit a2f70306df
156 changed files with 35085 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/density/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
build/
benchmark/build

6
contrib/density/.gitmodules vendored Normal file
View File

@ -0,0 +1,6 @@
[submodule "benchmark/libs/cputime"]
path = benchmark/libs/cputime
url = https://github.com/k0dai/cputime.git
[submodule "benchmark/libs/spookyhash"]
path = benchmark/libs/spookyhash
url = https://github.com/k0dai/spookyhash.git

272
contrib/density/.travis.yml Normal file
View File

@ -0,0 +1,272 @@
language: c
matrix:
fast_finish: true
include:
- os: osx
env:
- C_COMPILER=clang
- os: osx
env:
- C_COMPILER=gcc
- dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-arm
- qemu-user-static
- gcc-aarch64-linux-gnu
- g++-aarch64-linux-gnu
env:
- C_COMPILER=aarch64-linux-gnu-gcc MAKE_OPTIONS="AR=gcc-ar NATIVE=off" QEMU_INTERPRETER=qemu-aarch64-static PRE_SCRIPT="export QEMU_LD_PREFIX=/usr/aarch64-linux-gnu/"
- dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-arm
- qemu-user-static
- gcc-arm-linux-gnueabi
- g++-arm-linux-gnueabi
env:
- C_COMPILER=arm-linux-gnueabi-gcc MAKE_OPTIONS="AR=gcc-ar NATIVE=off" QEMU_INTERPRETER=qemu-arm-static PRE_SCRIPT="export QEMU_LD_PREFIX=/usr/arm-linux-gnueabi/"
- dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-ppc
- qemu-user-static
- gcc-powerpc-linux-gnu
- g++-powerpc-linux-gnu
env:
- C_COMPILER=powerpc-linux-gnu-gcc MAKE_OPTIONS="AR=gcc-ar ARCH=64" QEMU_INTERPRETER="qemu-ppc64-static" PRE_SCRIPT="export QEMU_LD_PREFIX=/usr/powerpc-linux-gnu/"
- dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-ppc
- qemu-user-static
- gcc-powerpc-linux-gnu
- g++-powerpc-linux-gnu
env:
- C_COMPILER=powerpc-linux-gnu-gcc MAKE_OPTIONS="AR=gcc-ar NATIVE=off" QEMU_INTERPRETER="qemu-ppc-static" PRE_SCRIPT="export QEMU_LD_PREFIX=/usr/powerpc-linux-gnu/"
- dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.8
env:
- C_COMPILER=gcc-4.8 MAKE_OPTIONS="AR=gcc-ar"
- dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.8
- gcc-4.8-multilib
env:
- C_COMPILER=gcc-4.8 MAKE_OPTIONS="AR=gcc-ar ARCH=32"
- dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.9
env:
- C_COMPILER=gcc-4.9 MAKE_OPTIONS="AR=gcc-ar"
- dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.9
- gcc-4.9-multilib
env:
- C_COMPILER=gcc-4.9 MAKE_OPTIONS="AR=gcc-ar ARCH=32"
- dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-5
env:
- C_COMPILER=gcc-5 MAKE_OPTIONS="AR=gcc-ar-5"
- dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-5
- gcc-5-multilib
env:
- C_COMPILER=gcc-5 MAKE_OPTIONS="AR=gcc-ar-5 ARCH=32"
# - dist: trusty
# sudo: required
# addons:
# apt:
# sources:
# - ubuntu-toolchain-r-test
# packages:
# - gcc-6
# env:
# - C_COMPILER=gcc-6 MAKE_OPTIONS="AR=gcc-ar-6"
- dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-6
- gcc-6-multilib
env:
- C_COMPILER=gcc-6 MAKE_OPTIONS="AR=gcc-ar-6 ARCH=32"
# - dist: trusty
# sudo: required
# addons:
# apt:
# sources:
# - ubuntu-toolchain-r-test
# packages:
# - gcc-7
# env:
# - C_COMPILER=gcc-7 MAKE_OPTIONS="AR=gcc-ar-7"
- dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-7
- gcc-7-multilib
env:
- C_COMPILER=gcc-7 MAKE_OPTIONS="AR=gcc-ar-7 ARCH=32"
- dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-trusty-3.5
packages:
- clang-3.5
- llvm-3.5-dev
env:
- C_COMPILER=clang-3.5 MAKE_OPTIONS="AR=llvm-ar-3.5" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld"
- dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-trusty-3.6
packages:
- clang-3.6
- llvm-3.6-dev
env:
- C_COMPILER=clang-3.6 MAKE_OPTIONS="AR=llvm-ar-3.6" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld"
# - dist: precise
# sudo: required
# addons:
# apt:
# sources:
# - ubuntu-toolchain-r-test
# - llvm-toolchain-precise-3.7
# packages:
# - clang-3.7
# - llvm-3.7-dev
# env:
# - C_COMPILER=clang-3.7 MAKE_OPTIONS="AR=llvm-ar-3.7" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld"
# - dist: precise
# sudo: required
# addons:
# apt:
# sources:
# - ubuntu-toolchain-r-test
# - llvm-toolchain-precise-3.8
# packages:
# - clang-3.8
# - llvm-3.8-dev
# env:
# - C_COMPILER=clang-3.8 MAKE_OPTIONS="AR=llvm-ar-3.8" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld"
- dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-trusty-3.9
packages:
- clang-3.9
- llvm-3.9-dev
env:
- C_COMPILER=clang-3.9 MAKE_OPTIONS="AR=llvm-ar-3.9" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld"
- dist: trusty
sudo: required
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-trusty-4.0
packages:
- clang-4.0
- llvm-4.0-dev
env:
- C_COMPILER=clang-4.0 MAKE_OPTIONS="AR=llvm-ar-4.0" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld"
# - dist: trusty
# sudo: required
# addons:
# apt:
# sources:
# - ubuntu-toolchain-r-test
# - llvm-toolchain-trusty-5.0
# packages:
# - clang-5.0
# - llvm-5.0-dev
# env:
# - C_COMPILER=clang-5.0 MAKE_OPTIONS="AR=llvm-ar-5.0" PRE_SCRIPT="sudo ln -sf ld.gold /usr/bin/ld"
script:
- ${PRE_SCRIPT}
- $C_COMPILER -v
- make CC=$C_COMPILER $MAKE_OPTIONS
- file ./build/*
- $QEMU_INTERPRETER ./build/benchmark -h -f

168
contrib/density/CHANGELOG.md Normal file
View File

@ -0,0 +1,168 @@
0.14.2
------
*February 12, 2018*
* Improved chameleon decode speed
* Added data hash checks and display option in benchmark
* Now using makefiles as build system
* Big endian support correctly implemented and tested
* Improved continuous integration tests
0.14.1
------
*January 20, 2018*
* Added MSVC support
* Added continuous integration on travis and appveyor
* Premake script improvement
* Various codebase improvements
0.14.0
------
*January 16, 2018*
* First stable version of DENSITY
* Complete project reorganization and API rewrite
* Many stability fixes and improvements
* Fast revert to conditional copy for incompressible input
* Custom dictionaries in API
* Improvements in compression ratio and speed
0.12.5 beta
-----------
*June 20, 2015*
* Added conditional main footer read/write
* Improved teleport staging buffer management
* Regression - a minimum buffer output size has to be ensured to avoid signature loss
* Modified the minimum lookahead and the resulting minimum buffer size in the API
* Lion : corrected a signature interception problem due to an increase in process unit size
* Lion : corrected chunk count conditions for new block / mode marker detection
* Lion : modified end of stream marker conditions
* Stability fixes and improvements
0.12.4 beta
-----------
*May 25, 2015*
* Removed remaining undefined behavior potential occurences
* Implemented parallelizable decompressible output block header reads/writes (disabled by default)
0.12.3 beta
-----------
*May 20, 2015*
* New lion algorithm, faster and more efficient
* Compiler specific optimizations
* Switched to premake 5 to benefit from link time optimizations
* Various fixes and improvements
0.12.2 beta
-----------
*May 4, 2015*
* Added an integrated in-memory benchmark
* Better Windows compatibility
* Fixed misaligned load/stores
* Switched to the premake build system
* Performance optimizations (pointers, branches, loops ...)
* Various fixes and improvements
0.12.1 beta
-----------
*April 3, 2015*
* Better unrolling readability and efficiency
* Improved read speed of dictionary/predictions entries
* Implemented case generators in cheetah to speed up decoding by using less branches
* Added signatures interception in lion to cancel the need for large output buffers
* Improved lion decode speed with specific form data access and use of ctz in form read
* Enabled decompression to exact-sized buffer for all algorithms
* Various fixes and improvements
0.12.0 beta
-----------
*March 24, 2015*
* Added new lion kernel
* Renamed kernel mandala to cheetah
* Kernel chameleon and cheetah improvements in encoding/decoding speeds
* Generic function macros to avoid code rewrite
* Improved memory teleport IO flexibility and speed, bytes issued by memory teleport can now be partially read
* Various fixes and improvements
0.11.3 beta
-----------
*February 5, 2015*
* Added integrity check system
* Corrected pointer usage and update on footer read/writes
* Now freeing kernel state memory only when compression mode is not copy
* Updated Makefiles
* Improved memory teleport
* Fixed sequencing problem after kernels request a new block
0.11.2 beta
-----------
*February 3, 2015*
* Added an algorithms overview in README
* Removed ssc references
* Now initializing last hash to zero on mandala kernel inits
* Reimplemented the buffer API
* Various corrections and improvements
0.11.1 beta
-----------
*January 19, 2015*
* Added a sharc benchmark in README
* Stateless memory teleport
* Improved event management and dispatching
* Improved compression/decompression finishes
* Improved streams API
* Various bug fixes, robustness improvements
0.10.2 beta
-----------
*January 7, 2015*
* Improved organization of compile-time switches and run-time options in the API
* Removed method density_stream_decompress_utilities_get_header from the API, header info is now returned in the density_stream_decompress_init function
* Corrected readme to reflect API changes
0.10.1 beta
-----------
*January 5, 2015*
* Re-added mandala kernel
* Corrected available bytes adjustment problem
* Added missing restrict keywords
* Cleaned unnecessary defines
0.10.0 beta
-----------
*January 2, 2015*
* Complete stream API redesign to greatly improve flexibility
* Only one supported algorithm for now : Chameleon
0.9.12 beta
-----------
*December 2, 2013*
* Mandala kernel addition, replacing dual pass chameleon
* Simplified, faster hash function
* Fixed memory freeing issue during main encoding/decoding finish
* Implemented no footer encode output type
* Namespace migration, kernel structure reorganization
* Corrected copy mode problem
* Implemented efficiency checks and mode reversions
* Corrected lack of main header parameters retrieval
* Fixed stream not being properly ended when mode reversion occurred
* Updated metadata computations
0.9.11 beta
-----------
*November 2, 2013*
* First beta release of DENSITY, including all the compression code from SHARC in a standalone, BSD licensed library
* Added copy mode (useful for enhancing data security via the density block checksums for example)
* Makefile produces static and dynamic libraries

28
contrib/density/LICENSE.md Normal file
View File

@ -0,0 +1,28 @@
Copyright (c) 2013, Guillaume Voirin
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

261
contrib/density/README.md Normal file
View File

@ -0,0 +1,261 @@
DENSITY
========
Superfast compression library
DENSITY is a free C99, open-source, BSD licensed compression library.
It is focused on high-speed compression, at the best ratio possible. **All three** of DENSITY's algorithms are currently at the **pareto frontier** of compression speed vs ratio (cf. [here](https://github.com/inikep/lzbench/blob/master/lzbench18_sorted.md) for an independent benchmark).
DENSITY features a simple API to enable quick integration in any project.
Branch|Linux & MacOS|Windows
--- | --- | ---
master|[![Build Status](https://travis-ci.org/k0dai/density.svg?branch=master)](https://travis-ci.org/k0dai/density)|[![Build status](https://ci.appveyor.com/api/projects/status/rf7x3x829il72cii/branch/master?svg=true)](https://ci.appveyor.com/project/gpnuma/density/branch/master)
dev|[![Build Status](https://travis-ci.org/k0dai/density.svg?branch=dev)](https://travis-ci.org/k0dai/density)|[![Build status](https://ci.appveyor.com/api/projects/status/rf7x3x829il72cii/branch/dev?svg=true)](https://ci.appveyor.com/project/gpnuma/density/branch/dev)
Why is it so fast ?
-------------------
One of the biggest assets of DENSITY is that its work unit is **not a byte** like other libraries, but **a group of 4 bytes**.
When other libraries consume one byte of data and then apply an algorithmic processing to it, DENSITY consumes 4 bytes and then applies its algorithmic processing.
That's why DENSITY's algorithms were designed from scratch. They have to alleviate for 4-byte work units and still provide interesting compression ratios.
**Speed pedigree traits**
* 4-byte work units
* heavy use of registers as opposed to memory for processing
* avoidance of or use of minimal branching when possible
* use of low memory data structures to favor processor cache Lx accesses
* library wide inlining
* specific unrollings
* prefetching and branching hints
* restricted pointers to maximize compiler optimizations
A "blowup protection" is provided, dramatically increasing the processing speed of incompressible input data. Also, the output, compressed data size will **never exceed** the original uncompressed data size by more than 1% in case of incompressible, reasonably-sized inputs.
Benchmarks
----------
**Quick benchmark**
DENSITY features an **integrated in-memory benchmark**. After building the project (see [build](#build)), a *benchmark* executable will be present in the build directory. If run without arguments, usage help will be displayed.
File used : enwik8 (100 MB)
Platform : MacBook Pro, MacOS 10.13.3, 2.3 GHz Intel Core i7, 8Gb 1600 MHz DDR, SSD, compiling with Clang/LLVM 9.0.0
Timing : using the *time* function, and taking the best *user* output after multiple runs. In the case of density, the in-memory integrated benchmark's best value (which uses the same usermode CPU timing) is used.
<sub>Library</sub>|<sub>Algorithm</sub>|<sub>Compress</sub>|<sub>Decompress</sub>|<sub>Size</sub>|<sub>Ratio</sub>|<sub>Round trip</sub>
---|---|---|---|---|---|---
<sub>**density** 0.14.2</sub>|<sub>Chameleon</sub>|<sub>0.092s (1085 MB/s)</sub>|<sub>0.059s (1684 MB/s)</sub>|<sub>61 524 084</sub>|<sub>61,52%</sub>|<sub>0.151s</sub>
<sub>lz4 r129</sub>|<sub>-1</sub>|<sub>0.468s (214 MB/s)</sub>|<sub>0.115s (870 MB/s)</sub>|<sub>57 285 990</sub>|<sub>57,29%</sub>|<sub>0.583s</sub>
<sub>lzo 2.08</sub>|<sub>-1</sub>|<sub>0.367s (272 MB/s)</sub>|<sub>0.309s (324 MB/s)</sub>|<sub>56 709 096</sub>|<sub>56,71%</sub>|<sub>0.676s</sub>
<sub>**density** 0.14.2</sub>|<sub>Cheetah</sub>|<sub>0.170s (587 MB/s)</sub>|<sub>0.126s (796 MB/s)</sub>|<sub>53 156 668</sub>|<sub>53,16%</sub>|<sub>0.296s</sub>
<sub>**density** 0.14.2</sub>|<sub>Lion</sub>|<sub>0.303s (330 MB/s)</sub>|<sub>0.288s (347 MB/s)</sub>|<sub>47 817 692</sub>|<sub>47,82%</sub>|<sub>0.591s</sub>
<sub>lz4 r129</sub>|<sub>-3</sub>|<sub>1.685s (59 MB/s)</sub>|<sub>0.118s (847 MB/s)</sub>|<sub>44 539 940</sub>|<sub>44,54%</sub>|<sub>1.803s</sub>
<sub>lzo 2.08</sub>|<sub>-7</sub>|<sub>9.562s (10 MB/s)</sub>|<sub>0.319s (313 MB/s)</sub>|<sub>41 720 721</sub>|<sub>41,72%</sub>|<sub>9.881s</sub>
**Other benchmarks**
Here are a few other benchmarks featuring DENSITY (non exhaustive list) :
* [**squash**](https://github.com/quixdb/squash) is an abstraction layer for compression algorithms, and has an extremely exhaustive set of benchmark results, including density's, [available here](https://quixdb.github.io/squash-benchmark/?dataset=dickens&machine=s-desktop).
* [**lzbench**](https://github.com/inikep/lzbench) is an in-memory benchmark of open-source LZ77/LZSS/LZMA compressors.
* [**fsbench**](https://github.com/gpnuma/fsbench-density) is a command line utility that enables real-time testing of compression algorithms, but also hashes and much more. A fork with density releases is [available here](https://github.com/gpnuma/fsbench-density) for easy access.
The original author's repository [can be found here](https://chiselapp.com/user/Justin_be_my_guide/repository/fsbench/).
Build
-----
DENSITY can be built on a number of platforms, via the provided makefiles.
It was developed and optimized against Clang/LLVM which makes it the preferred compiler, but GCC and MSVC are also supported. Please use the latest compiler versions for best performance.
**MacOS**
On MacOS, Clang/LLVM is the default compiler, which makes things simpler.
1) Get the source code :
```
git clone https://github.com/k0dai/density.git
cd density
```
2) Build and test :
```
make
build/benchmark -f
```
Alternatively, thanks to the [Homebrew project](https://brew.sh), DENSITY can also be installed with a single command on MacOS:
```
brew install density
```
**Linux**
On Linux, Clang/LLVM is not always available by default, but can be easily added thanks to the provided package managers.
The following example assumes a Debian or Ubuntu distribution with *apt-get*.
1) From the command line, install Clang/LLVM (*optional*, GCC is also supported if Clang/LLVM can't be used) and other prerequisites.
```
sudo apt-get install clang git
```
2) Get the source code :
```
git clone https://github.com/k0dai/density.git
cd density
```
3) Build and test :
```
make
```
or
```
make CC=gcc-... AR=gcc-ar-...
```
or
```
make CC=clang-... AR=llvm-ar-...
```
to choose alternative compilers. For a quick test of resulting binaries, run
```
build/benchmark -f
```
**Windows**
Please install [git for Windows](https://git-scm.com/download/win) to begin with.
On Windows, density can be built in different ways.
The **first method** is to use mingw's gcc compiler; for that it is necessary to download and install [mingw-w64](https://sourceforge.net/projects/mingw-w64/).
1) Once mingw-w64 is installed, get the source :
```
git clone https://github.com/k0dai/density.git
cd density
```
2) Build and test :
```
mingw32-make.exe
build/benchmark.exe -f
```
As an alternative, [MSYS2](http://www.msys2.org/) also offers a linux-like environment for Windows.
The **second method** is to download and install Microsoft's [Visual Studio IDE community edition](https://www.visualstudio.com/thank-you-downloading-visual-studio/?sku=Community). It comes with Microsoft's own compilers and is free.
1) Once Visual Studio is installed, open a [developer command prompt](https://docs.microsoft.com/en-us/dotnet/framework/tools/developer-command-prompt-for-vs) and type :
```
git clone https://github.com/k0dai/density.git
cd density\msvc
```
2) Build and test :
```
msbuild Density.sln
bin\Release\benchmark.exe -f
```
An extra **recommended step** would be to install *Clang/LLVM* for Windows. It is downloadable from [this link](http://releases.llvm.org/5.0.1/LLVM-5.0.1-win64.exe). Once installed, open the Visual Studio IDE by double-clicking on *Density.sln*, then right-click on project names and change the platform toolsets to *LLVM*. Rebuild the solution to generate binaries with Clang/LLVM.
Output format
-------------
DENSITY outputs compressed data in a simple format, which enables file storage and optional parallelization for both compression and decompression.
A very short header holding vital informations (like DENSITY version and algorithm used) precedes the binary compressed data.
APIs
----
DENSITY features a straightforward *API*, simple yet powerful enough to keep users' creativity unleashed.
For advanced developers, it allows use of custom dictionaries and exportation of generated dictionaries after a compression session. Although using the default, blank dictionary is perfectly fine in most cases, setting up your own, tailored dictionaries could somewhat improve compression ratio especially for low sized input datum.
Please see the [*quick start*](#quick-start-a-simple-example-using-the-api) at the bottom of this page.
About the algorithms
--------------------
**Chameleon** ( *DENSITY_ALGORITHM_CHAMELEON* )
Chameleon is a dictionary lookup based compression algorithm. It is designed for absolute speed and usually reaches a 60% compression ratio on compressible data.
Decompression is just as fast. This algorithm is a great choice when main concern is speed.
**Cheetah** ( *DENSITY_ALGORITHM_CHEETAH* )
Cheetah was developed with inputs from [Piotr Tarsa](https://github.com/tarsa).
It is derived from chameleon and uses swapped double dictionary lookups and predictions. It can be extremely good with highly compressible data (ratio reaching 10% or less).
On typical compressible data compression ratio is about 50% or less. It is still extremely fast for both compression and decompression and is a great, efficient all-rounder algorithm.
**Lion** ( *DENSITY_ALGORITHM_LION* )
Lion is a multiform compression algorithm derived from cheetah. It goes further in the areas of dynamic adaptation and fine-grained analysis.
It uses multiple swapped dictionary lookups and predictions, and forms rank entropy coding.
Lion provides the best compression ratio of all three algorithms under any circumstance, and is still very fast.
Quick start (a simple example using the API)
--------------------------------------------
Using DENSITY in your application couldn't be any simpler.
First you need to include this file in your project :
* density_api.h
When this is done you can start using the **DENSITY API** :
```C
#include <string.h>
#include "density_api.h"
char* text = "This is a simple example on how to use the simple Density API. This is a simple example on how to use the simple Density API.";
uint64_t text_length = (uint64_t)strlen(text);
// Determine safe buffer sizes
uint_fast64_t compress_safe_size = density_compress_safe_size(text_length);
uint_fast64_t decompress_safe_size = density_decompress_safe_size(text_length);
// Allocate required memory
uint8_t *outCompressed = malloc(compress_safe_size * sizeof(char));
uint8_t *outDecompressed = malloc(decompress_safe_size * sizeof(char));
density_processing_result result;
// Compress
result = density_compress(text, text_length, outCompressed, compress_safe_size, DENSITY_COMPRESSION_MODE_CHAMELEON_ALGORITHM);
if(!result.state)
printf("Compressed %llu bytes to %llu bytes\n", result.bytesRead, result.bytesWritten);
// Decompress
result = density_decompress(outCompressed, result.bytesWritten, outDecompressed, decompress_safe_size);
if(!result.state)
printf("Decompressed %llu bytes to %llu bytes\n", result.bytesRead, result.bytesWritten);
// Free memory_allocated
free(outCompressed);
free(outDecompressed);
```
And that's it ! We've done a compression/decompression round trip with a few lines !
Related projects
----------------
* **SHARC** (archiver using density algorithms) [https://github.com/gpnuma/sharc](https://github.com/gpnuma/sharc)
* **fsbench-density** (in-memory transformations benchmark) [https://github.com/gpnuma/fsbench-density](https://github.com/gpnuma/fsbench-density)
* **densityxx** (c++ port of density) [https://github.com/charlesw1234/densityxx](https://github.com/charlesw1234/densityxx)

37
contrib/density/appveyor.yml Normal file
View File

@ -0,0 +1,37 @@
version: 0.14.2.{build}
image: Visual Studio 2015
init:
- cmd:
environment:
matrix:
- EXTRA_PATH: C:\mingw-w64\i686-5.3.0-posix-dwarf-rt_v4-rev0\mingw32\bin\
LAUNCH_BUILD: mingw32-make CC=gcc AR=gcc-ar
LAUNCH_PATH: build\
- EXTRA_PATH: C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\
LAUNCH_BUILD: mingw32-make CC=gcc AR=gcc-ar
LAUNCH_PATH: build\
- EXTRA_PATH: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\
LAUNCH_BUILD: mingw32-make CC=gcc AR=gcc-ar
LAUNCH_PATH: build\
- EXTRA_PATH: C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin\
LAUNCH_BUILD: mingw32-make CC=gcc AR=gcc-ar
LAUNCH_PATH: build\
- EXTRA_PATH: '%programfiles(x86)%\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\amd64\'
LAUNCH_BUILD: cd msvc && msbuild.exe Density.sln
EXTRA_GIT: git submodule update --init --recursive
LAUNCH_PATH: bin\Release\
install:
- cmd:
build_script:
- cmd: >-
%EXTRA_GIT%
set PATH=%EXTRA_PATH%;%PATH%
%LAUNCH_BUILD%
file %LAUNCH_PATH:\=/%*
%LAUNCH_PATH%benchmark.exe -h -f
test_script:
- cmd:

345
contrib/density/benchmark/src/benchmark.c Normal file
View File

@ -0,0 +1,345 @@
/*
* Density benchmark
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* 5/04/15 19:30
*/
#include "benchmark.h"
void density_benchmark_version() {
printf("\nSingle threaded ");
DENSITY_BENCHMARK_BOLD(printf("in-memory benchmark"));
printf(" powered by ");
DENSITY_BENCHMARK_BOLD(printf("Density %i.%i.%i\n", density_version_major(), density_version_minor(), density_version_revision()));
printf("Copyright (C) 2015 Guillaume Voirin\n");
printf("Built for %s (%s endian system, %u bits) using " DENSITY_BENCHMARK_COMPILER ", %s %s\n", DENSITY_BENCHMARK_PLATFORM_STRING, DENSITY_BENCHMARK_ENDIAN_STRING, (unsigned int) (8 * sizeof(void *)), DENSITY_BENCHMARK_COMPILER_VERSION, __DATE__, __TIME__);
}
void density_benchmark_client_usage() {
printf("\n");
DENSITY_BENCHMARK_BOLD(printf("Usage :\n"));
printf(" benchmark [OPTIONS ?]... [FILE ?]\n\n");
DENSITY_BENCHMARK_BOLD(printf("Available options :\n"));
printf(" -[LEVEL] Test file using only the specified compression LEVEL\n");
printf(" If unspecified, all algorithms are tested (default).\n");
printf(" LEVEL can have the following values (as values become higher,\n");
printf(" compression ratio increases and speed diminishes) :\n");
printf(" 0 = Copy (no compression)\n");
printf(" 1 = Chameleon algorithm\n");
printf(" 2 = Cheetah algorithm\n");
printf(" 3 = Lion algorithm\n");
printf(" -c Compress only\n");
printf(" -f Activate fuzzer mode (pseudorandom generated data)\n");
printf(" -h Print data hashing informations\n\n");
exit(EXIT_SUCCESS);
}
void density_benchmark_format_decimal(uint64_t number) {
if (number < 1000) {
printf("%"PRIu64, number);
return;
}
density_benchmark_format_decimal(number / 1000);
printf(",%03"PRIu64, number % 1000);
}
const char *density_benchmark_convert_state_to_text(DENSITY_STATE state) {
switch (state) {
case DENSITY_STATE_ERROR_DURING_PROCESSING:
return "Error during processing";
case DENSITY_STATE_ERROR_INPUT_BUFFER_TOO_SMALL:
return "Input buffer is too small";
case DENSITY_STATE_ERROR_OUTPUT_BUFFER_TOO_SMALL:
return "Output buffer is too small";
case DENSITY_STATE_ERROR_INVALID_CONTEXT:
return "Invalid context";
case DENSITY_STATE_ERROR_INVALID_ALGORITHM:
return "Invalid algorithm";
default:
return "Unknown error";
}
}
int main(int argc, char *argv[]) {
density_benchmark_version();
DENSITY_ALGORITHM start_mode = DENSITY_ALGORITHM_CHAMELEON;
DENSITY_ALGORITHM end_mode = DENSITY_ALGORITHM_LION;
bool compression_only = false;
bool fuzzer = false;
bool hash_info = false;
char *file_path = NULL;
if (argc <= 1)
density_benchmark_client_usage();
for (int count = 1; count < argc; count++) {
if (argv[count][0] == '-') {
switch (argv[count][1]) {
case '1':
start_mode = DENSITY_ALGORITHM_CHAMELEON;
end_mode = DENSITY_ALGORITHM_CHAMELEON;
break;
case '2':
start_mode = DENSITY_ALGORITHM_CHEETAH;
end_mode = DENSITY_ALGORITHM_CHEETAH;
break;
case '3':
start_mode = DENSITY_ALGORITHM_LION;
end_mode = DENSITY_ALGORITHM_LION;
break;
case 'c':
compression_only = true;
break;
case 'f':
fuzzer = true;
break;
case 'h':
hash_info = true;
break;
default:
density_benchmark_client_usage();
}
} else
file_path = argv[argc - 1];
}
uint8_t *in;
uint8_t *out;
uint_fast64_t uncompressed_size;
uint_fast64_t memory_allocated;
if (fuzzer) {
srand((unsigned int) (time(NULL) * 14521937821257379531llu));
uncompressed_size = (uint_fast64_t) (((uint64_t) (rand() * 100000000llu)) / RAND_MAX);
memory_allocated = density_compress_safe_size(uncompressed_size);
in = malloc(memory_allocated * sizeof(uint8_t));
uint8_t value = (uint8_t) rand();
for (unsigned int count = 0; count < uncompressed_size; count++) {
if (!(rand() & 0xf))
value += (uint8_t)rand();
in[count] = value;
}
out = malloc(memory_allocated * sizeof(uint8_t));
} else {
// Open file and get infos
FILE *file = fopen(file_path, "rb");
if (file == NULL) {
DENSITY_BENCHMARK_ERROR(printf("Error opening file %s.", file_path), false);
}
struct stat file_attributes;
stat(file_path, &file_attributes);
// Allocate memory and copy file to memory
uncompressed_size = (uint_fast64_t) file_attributes.st_size;
memory_allocated = density_compress_safe_size(uncompressed_size);
in = malloc(memory_allocated * sizeof(uint8_t));
size_t read = fread(in, sizeof(uint8_t), uncompressed_size, file);
if(uncompressed_size != read) {
DENSITY_BENCHMARK_ERROR(printf("Error reading file %s.", file_path), false);
}
fclose(file);
out = malloc(memory_allocated * sizeof(uint8_t));
}
printf("Allocated ");
density_benchmark_format_decimal(2 * memory_allocated);
printf(" bytes of in-memory work space\n");
uint64_t original_hash_1 = DENSITY_BENCHMARK_HASH_SEED_1;
uint64_t original_hash_2 = DENSITY_BENCHMARK_HASH_SEED_2;
spookyhash_128(in, uncompressed_size, &original_hash_1, &original_hash_2);
printf("\n");
for (DENSITY_ALGORITHM compression_mode = start_mode; compression_mode <= end_mode; compression_mode++) {
// Print algorithm info
switch (compression_mode) {
case DENSITY_ALGORITHM_CHAMELEON:
DENSITY_BENCHMARK_BLUE(DENSITY_BENCHMARK_BOLD(printf("Chameleon algorithm")));
DENSITY_BENCHMARK_UNDERLINE(19);
break;
case DENSITY_ALGORITHM_CHEETAH:
DENSITY_BENCHMARK_BLUE(DENSITY_BENCHMARK_BOLD(printf("Cheetah algorithm")));
DENSITY_BENCHMARK_UNDERLINE(17);
break;
case DENSITY_ALGORITHM_LION:
DENSITY_BENCHMARK_BLUE(DENSITY_BENCHMARK_BOLD(printf("Lion algorithm")));
DENSITY_BENCHMARK_UNDERLINE(14);
break;
}
fflush(stdout);
// Pre-heat
printf("\nUsing ");
if (fuzzer) {
DENSITY_BENCHMARK_BOLD(printf("generated data"));
} else {
printf("file ");
DENSITY_BENCHMARK_BOLD(printf("%s", file_path));
}
printf(" copied in memory\n");
if(hash_info) {
printf("Uncompressed data hash is ");
DENSITY_BENCHMARK_BOLD(printf("0x%" PRIx64 "%" PRIx64, original_hash_1, original_hash_2));
printf("\n");
}
density_processing_result result = density_compress(in, uncompressed_size, out, memory_allocated, compression_mode);
if (result.state) {
DENSITY_BENCHMARK_ERROR(printf("During compress API returned error %i (%s).", result.state, density_benchmark_convert_state_to_text(result.state)), true);
}
const uint_fast64_t compressed_size = result.bytesWritten;
uint64_t hash_1 = DENSITY_BENCHMARK_HASH_SEED_1;
uint64_t hash_2 = DENSITY_BENCHMARK_HASH_SEED_2;
if(hash_info) {
spookyhash_128(out, compressed_size, &hash_1, &hash_2);
printf("Compressed data hash is ");
DENSITY_BENCHMARK_BOLD(printf("0x%" PRIx64 "%" PRIx64, hash_1, hash_2));
printf("\n");
}
if (!compression_only) {
memset(in, 0, memory_allocated);
result = density_decompress(out, compressed_size, in, memory_allocated);
if (result.state) {
DENSITY_BENCHMARK_ERROR(printf("During decompress API returned error %i (%s).", result.state, density_benchmark_convert_state_to_text(result.state)), true);
}
if (result.bytesWritten != uncompressed_size) {
DENSITY_BENCHMARK_ERROR(printf("Round-trip size differs from original size (");
density_benchmark_format_decimal(result.bytesWritten);
printf(" bytes against ");
density_benchmark_format_decimal(uncompressed_size);
printf(" bytes).");, true);
}
hash_1 = DENSITY_BENCHMARK_HASH_SEED_1;
hash_2 = DENSITY_BENCHMARK_HASH_SEED_2;
spookyhash_128(in, uncompressed_size, &hash_1, &hash_2);
if(hash_info) {
printf("Round-trip data hash is ");
DENSITY_BENCHMARK_BOLD(printf("0x%" PRIx64 "%" PRIx64, hash_1, hash_2));
printf("\n");
}
if(hash_1 != original_hash_1 || hash_2 != original_hash_2) {
DENSITY_BENCHMARK_ERROR(printf("Uncompressed and round-trip data hashes do not match (");
printf("0x%" PRIx64 "%" PRIx64, hash_1, hash_2);
printf(" vs. ");
printf("0x%" PRIx64 "%" PRIx64, original_hash_1, original_hash_2);
printf(").");, true);
}
printf("Uncompressed and round-trip data hashes match. ");
}
printf("Starting main bench.\n");
if (compression_only)
printf("Compressing ");
else
printf("Round-tripping ");
density_benchmark_format_decimal(uncompressed_size);
printf(" bytes to ");
density_benchmark_format_decimal(compressed_size);
printf(" bytes (compression ratio ");
DENSITY_BENCHMARK_BOLD(printf("%.2lf%%", (100.0 * compressed_size) / uncompressed_size));
printf(" or ");
DENSITY_BENCHMARK_BOLD(printf("%.3fx", (1.0 * uncompressed_size) / compressed_size));
if (compression_only)
printf(")\n");
else
printf(") and back\n");
fflush(stdout);
// Main benchmark
unsigned int iterations = 0;
double compress_time_high = 0.0;
double compress_time_low = 60.0;
double decompress_time_high = 0.0;
double decompress_time_low = 60.0;
double total_compress_time = 0.0;
double total_decompress_time = 0.0;
double total_time = 0.0;
double decompress_speed = 0.0;
double decompress_speed_low = 0.0;
double decompress_speed_high = 0.0;
double compress_time_elapsed = 0.0;
double decompress_time_elapsed = 0.0;
cputime_chronometer chrono;
while (total_time <= 10.0) {
++iterations;
cputime_chronometer_start(&chrono);
density_compress(in, uncompressed_size, out, memory_allocated, compression_mode);
compress_time_elapsed = cputime_chronometer_stop(&chrono);
if (!compression_only) {
cputime_chronometer_start(&chrono);
density_decompress(out, compressed_size, in, memory_allocated);
decompress_time_elapsed = cputime_chronometer_stop(&chrono);
}
total_compress_time += compress_time_elapsed;
if (compress_time_elapsed < compress_time_low)
compress_time_low = compress_time_elapsed;
if (compress_time_elapsed > compress_time_high)
compress_time_high = compress_time_elapsed;
double compress_speed = ((1.0 * uncompressed_size * iterations) / (total_compress_time * 1000.0 * 1000.0));
double compress_speed_low = ((1.0 * uncompressed_size) / (compress_time_high * 1000.0 * 1000.0));
double compress_speed_high = ((1.0 * uncompressed_size) / (compress_time_low * 1000.0 * 1000.0));
total_time += compress_time_elapsed;
if (!compression_only) {
total_decompress_time += decompress_time_elapsed;
if (decompress_time_elapsed < decompress_time_low)
decompress_time_low = decompress_time_elapsed;
if (decompress_time_elapsed > decompress_time_high)
decompress_time_high = decompress_time_elapsed;
decompress_speed = ((1.0 * uncompressed_size * iterations) / (total_decompress_time * 1000.0 * 1000.0));
decompress_speed_low = ((1.0 * uncompressed_size) / (decompress_time_high * 1000.0 * 1000.0));
decompress_speed_high = ((1.0 * uncompressed_size) / (decompress_time_low * 1000.0 * 1000.0));
total_time += decompress_time_elapsed;
}
DENSITY_BENCHMARK_BLUE(printf("\rCompress speed ");
DENSITY_BENCHMARK_BOLD(printf("%.0lf MB/s", compress_speed)));
printf(" (min %.0lf MB/s, max %.0lf MB/s, best %.4lfs) ", compress_speed_low, compress_speed_high, compress_time_low);
if (!compression_only) {
printf("<=> ");
DENSITY_BENCHMARK_BLUE(printf("Decompress speed ");
DENSITY_BENCHMARK_BOLD(printf("%.0lf MB/s", decompress_speed)));
printf(" (min %.0lf MB/s, max %.0lf MB/s, best %.4lfs) ", decompress_speed_low, decompress_speed_high, decompress_time_low);
}
fflush(stdout);
}
printf("\nRun time %.3lfs (%i iterations)\n\n", total_time, iterations);
}
free(in);
free(out);
printf("Allocated memory released.\n\n");
return EXIT_SUCCESS;
}

129
contrib/density/benchmark/src/benchmark.h Normal file
View File

@ -0,0 +1,129 @@
/*
* Density benchmark
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* 6/04/15 0:11
*/
#ifndef DENSITY_BENCHMARK_H
#define DENSITY_BENCHMARK_H
#define _CRT_SECURE_NO_DEPRECATE
#include <sys/stat.h>
#include <sys/types.h>
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <stdlib.h>
#include <inttypes.h>
#include <time.h>
#include "../../src/density_api.h"
#include "../libs/cputime/src/cputime_api.h"
#include "../libs/spookyhash/src/spookyhash_api.h"
#if defined(_WIN64) || defined(_WIN32)
#else
#define DENSITY_BENCHMARK_ALLOW_ANSI_ESCAPE_SEQUENCES
#endif
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define DENSITY_BENCHMARK_ENDIAN_STRING "Little"
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#define DENSITY_BENCHMARK_ENDIAN_STRING "Big"
#endif
#if defined(__clang__)
#define DENSITY_BENCHMARK_COMPILER "Clang %d.%d.%d"
#define DENSITY_BENCHMARK_COMPILER_VERSION __clang_major__, __clang_minor__, __clang_patchlevel__
#elif defined(__GNUC__)
#define DENSITY_BENCHMARK_COMPILER "GCC %d.%d.%d"
#define DENSITY_BENCHMARK_COMPILER_VERSION __GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__
#elif defined(_MSC_VER)
#define DENSITY_BENCHMARK_COMPILER "MSVC"
#define DENSITY_BENCHMARK_COMPILER_VERSION ""
#elif defined(__INTEL_COMPILER)
#define DENSITY_BENCHMARK_COMPILER "ICC"
#define DENSITY_BENCHMARK_COMPILER_VERSION ""
#else
#define DENSITY_BENCHMARK_COMPILER "an unknown compiler"
#define DENSITY_BENCHMARK_COMPILER_VERSION ""
#endif
#if defined(_WIN64) || defined(_WIN32)
#define DENSITY_BENCHMARK_PLATFORM_STRING "Microsoft Windows"
#elif defined(__APPLE__)
#include "TargetConditionals.h"
#if TARGET_IPHONE_SIMULATOR
#define DENSITY_BENCHMARK_PLATFORM_STRING "iOS Simulator"
#elif TARGET_OS_IPHONE
#define DENSITY_BENCHMARK_PLATFORM_STRING "iOS"
#elif TARGET_OS_MAC
#define DENSITY_BENCHMARK_PLATFORM_STRING "MacOS"
#else
#define DENSITY_BENCHMARK_PLATFORM_STRING "an unknown Apple platform"
#endif
#elif defined(__FreeBSD__)
#define DENSITY_BENCHMARK_PLATFORM_STRING "FreeBSD"
#elif defined(__linux__)
#define DENSITY_BENCHMARK_PLATFORM_STRING "GNU/Linux"
#elif defined(__unix__)
#define DENSITY_BENCHMARK_PLATFORM_STRING "Unix"
#elif defined(__posix__)
#define DENSITY_BENCHMARK_PLATFORM_STRING "Posix"
#else
#define DENSITY_BENCHMARK_PLATFORM_STRING "an unknown platform"
#endif
#define DENSITY_ESCAPE_CHARACTER ((char)27)
#ifdef DENSITY_BENCHMARK_ALLOW_ANSI_ESCAPE_SEQUENCES
#define DENSITY_BENCHMARK_BOLD(op) printf("%c[1m", DENSITY_ESCAPE_CHARACTER);\
op;\
printf("%c[0m", DENSITY_ESCAPE_CHARACTER);
#define DENSITY_BENCHMARK_BLUE(op) printf("%c[0;34m", DENSITY_ESCAPE_CHARACTER);\
op;\
printf("%c[0m", DENSITY_ESCAPE_CHARACTER);
#define DENSITY_BENCHMARK_RED(op) printf("%c[0;31m", DENSITY_ESCAPE_CHARACTER);\
op;\
printf("%c[0m", DENSITY_ESCAPE_CHARACTER);
#else
#define DENSITY_BENCHMARK_BOLD(op) op;
#define DENSITY_BENCHMARK_BLUE(op) op;
#define DENSITY_BENCHMARK_RED(op) op;
#endif
#define DENSITY_BENCHMARK_UNDERLINE(n) printf("\n");\
for(int i = 0; i < n; i++) printf("=");
#define DENSITY_BENCHMARK_ERROR(op, issue) DENSITY_BENCHMARK_RED(DENSITY_BENCHMARK_BOLD(printf("\nAn error has occured !\n")));\
op;\
printf("\n");\
if(issue) {\
printf("Please open an issue at <https://github.com/k0dai/density/issues>, with your platform information and any relevant file.\n");\
DENSITY_BENCHMARK_BOLD(printf("Thank you !\n"));\
}\
fflush(stdout);\
exit(EXIT_FAILURE);
#endif
#define DENSITY_BENCHMARK_HASH_SEED_1 0x0123456789abcdefllu
#define DENSITY_BENCHMARK_HASH_SEED_2 0xfedcba9876543210llu

21
contrib/density/msvc/Density.sln Normal file
View File

@ -0,0 +1,21 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 14
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "density", "density.vcxproj", "{65C51F09-D1A4-9EA4-DABC-297B461B0506}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "benchmark", "benchmark.vcxproj", "{7000C5C1-DC6A-7938-25A9-2ADE9152578D}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{65C51F09-D1A4-9EA4-DABC-297B461B0506}.Release|x64.ActiveCfg = Release|x64
{65C51F09-D1A4-9EA4-DABC-297B461B0506}.Release|x64.Build.0 = Release|x64
{7000C5C1-DC6A-7938-25A9-2ADE9152578D}.Release|x64.ActiveCfg = Release|x64
{7000C5C1-DC6A-7938-25A9-2ADE9152578D}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

78
contrib/density/msvc/benchmark.vcxproj Normal file
View File

@ -0,0 +1,78 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{7000C5C1-DC6A-7938-25A9-2ADE9152578D}</ProjectGuid>
<IgnoreWarnCompileDuplicatedFilename>true</IgnoreWarnCompileDuplicatedFilename>
<Keyword>Win32Proj</Keyword>
<RootNamespace>benchmark</RootNamespace>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>bin\Release\</OutDir>
<IntDir>obj\Release\benchmark\</IntDir>
<TargetName>benchmark</TargetName>
<TargetExt>.exe</TargetExt>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<MinimalRebuild>false</MinimalRebuild>
<OmitFramePointers>true</OmitFramePointers>
<StringPooling>true</StringPooling>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="..\benchmark\libs\cputime\src\chronometer.h" />
<ClInclude Include="..\benchmark\libs\cputime\src\cputime_api.h" />
<ClInclude Include="..\benchmark\libs\spookyhash\src\context.h" />
<ClInclude Include="..\benchmark\libs\spookyhash\src\globals.h" />
<ClInclude Include="..\benchmark\libs\spookyhash\src\spookyhash.h" />
<ClInclude Include="..\benchmark\libs\spookyhash\src\spookyhash_api.h" />
<ClInclude Include="..\benchmark\src\benchmark.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\benchmark\libs\cputime\src\chronometer.c" />
<ClCompile Include="..\benchmark\libs\spookyhash\src\context.c" />
<ClCompile Include="..\benchmark\libs\spookyhash\src\globals.c" />
<ClCompile Include="..\benchmark\libs\spookyhash\src\spookyhash.c" />
<ClCompile Include="..\benchmark\src\benchmark.c" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="density.vcxproj">
<Project>{65C51F09-D1A4-9EA4-DABC-297B461B0506}</Project>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

63
contrib/density/msvc/benchmark.vcxproj.filters Normal file
View File

@ -0,0 +1,63 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="libs">
<UniqueIdentifier>{2F149A7C-1B4B-9B0D-C437-8110B04D170F}</UniqueIdentifier>
</Filter>
<Filter Include="libs\cputime">
<UniqueIdentifier>{95C9EAC4-812C-7A69-2AB2-B21F16F445EC}</UniqueIdentifier>
</Filter>
<Filter Include="libs\cputime\src">
<UniqueIdentifier>{2C4ACB69-1843-EABB-4175-CF402DCDC9C7}</UniqueIdentifier>
</Filter>
<Filter Include="libs\spookyhash">
<UniqueIdentifier>{C78AF53E-3316-6303-3C27-E6F7A831BF03}</UniqueIdentifier>
</Filter>
<Filter Include="libs\spookyhash\src">
<UniqueIdentifier>{5E54D780-CAB5-B48E-5323-FB40BF83EB4D}</UniqueIdentifier>
</Filter>
<Filter Include="src">
<UniqueIdentifier>{2DAB880B-99B4-887C-2230-9F7C8E38947C}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\benchmark\libs\cputime\src\chronometer.h">
<Filter>libs\cputime\src</Filter>
</ClInclude>
<ClInclude Include="..\benchmark\libs\cputime\src\cputime_api.h">
<Filter>libs\cputime\src</Filter>
</ClInclude>
<ClInclude Include="..\benchmark\libs\spookyhash\src\context.h">
<Filter>libs\spookyhash\src</Filter>
</ClInclude>
<ClInclude Include="..\benchmark\libs\spookyhash\src\globals.h">
<Filter>libs\spookyhash\src</Filter>
</ClInclude>
<ClInclude Include="..\benchmark\libs\spookyhash\src\spookyhash.h">
<Filter>libs\spookyhash\src</Filter>
</ClInclude>
<ClInclude Include="..\benchmark\libs\spookyhash\src\spookyhash_api.h">
<Filter>libs\spookyhash\src</Filter>
</ClInclude>
<ClInclude Include="..\benchmark\src\benchmark.h">
<Filter>src</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\benchmark\libs\cputime\src\chronometer.c">
<Filter>libs\cputime\src</Filter>
</ClCompile>
<ClCompile Include="..\benchmark\libs\spookyhash\src\context.c">
<Filter>libs\spookyhash\src</Filter>
</ClCompile>
<ClCompile Include="..\benchmark\libs\spookyhash\src\globals.c">
<Filter>libs\spookyhash\src</Filter>
</ClCompile>
<ClCompile Include="..\benchmark\libs\spookyhash\src\spookyhash.c">
<Filter>libs\spookyhash\src</Filter>
</ClCompile>
<ClCompile Include="..\benchmark\src\benchmark.c">
<Filter>src</Filter>
</ClCompile>
</ItemGroup>
</Project>

93
contrib/density/msvc/density.vcxproj Normal file
View File

@ -0,0 +1,93 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{65C51F09-D1A4-9EA4-DABC-297B461B0506}</ProjectGuid>
<IgnoreWarnCompileDuplicatedFilename>true</IgnoreWarnCompileDuplicatedFilename>
<Keyword>Win32Proj</Keyword>
<RootNamespace>density</RootNamespace>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>bin\Release\</OutDir>
<IntDir>obj\Release\density\</IntDir>
<TargetName>density</TargetName>
<TargetExt>.dll</TargetExt>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<MinimalRebuild>false</MinimalRebuild>
<OmitFramePointers>true</OmitFramePointers>
<StringPooling>true</StringPooling>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<ImportLibrary>bin\Release\density.lib</ImportLibrary>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="..\src\algorithms\algorithms.h" />
<ClInclude Include="..\src\algorithms\chameleon\chameleon.h" />
<ClInclude Include="..\src\algorithms\chameleon\core\chameleon_decode.h" />
<ClInclude Include="..\src\algorithms\chameleon\core\chameleon_encode.h" />
<ClInclude Include="..\src\algorithms\chameleon\dictionary\chameleon_dictionary.h" />
<ClInclude Include="..\src\algorithms\cheetah\cheetah.h" />
<ClInclude Include="..\src\algorithms\cheetah\core\cheetah_decode.h" />
<ClInclude Include="..\src\algorithms\cheetah\core\cheetah_encode.h" />
<ClInclude Include="..\src\algorithms\cheetah\dictionary\cheetah_dictionary.h" />
<ClInclude Include="..\src\algorithms\dictionaries.h" />
<ClInclude Include="..\src\algorithms\lion\core\lion_decode.h" />
<ClInclude Include="..\src\algorithms\lion\core\lion_encode.h" />
<ClInclude Include="..\src\algorithms\lion\dictionary\lion_dictionary.h" />
<ClInclude Include="..\src\algorithms\lion\forms\lion_form_model.h" />
<ClInclude Include="..\src\algorithms\lion\lion.h" />
<ClInclude Include="..\src\buffers\buffer.h" />
<ClInclude Include="..\src\density_api.h" />
<ClInclude Include="..\src\globals.h" />
<ClInclude Include="..\src\structure\header.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\src\algorithms\algorithms.c" />
<ClCompile Include="..\src\algorithms\chameleon\core\chameleon_decode.c" />
<ClCompile Include="..\src\algorithms\chameleon\core\chameleon_encode.c" />
<ClCompile Include="..\src\algorithms\cheetah\core\cheetah_decode.c" />
<ClCompile Include="..\src\algorithms\cheetah\core\cheetah_encode.c" />
<ClCompile Include="..\src\algorithms\dictionaries.c" />
<ClCompile Include="..\src\algorithms\lion\core\lion_decode.c" />
<ClCompile Include="..\src\algorithms\lion\core\lion_encode.c" />
<ClCompile Include="..\src\algorithms\lion\forms\lion_form_model.c" />
<ClCompile Include="..\src\buffers\buffer.c" />
<ClCompile Include="..\src\globals.c" />
<ClCompile Include="..\src\structure\header.c" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

135
contrib/density/msvc/density.vcxproj.filters Normal file
View File

@ -0,0 +1,135 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="algorithms">
<UniqueIdentifier>{DF10C562-CBC8-06B0-34D0-DF0B20A7A1A0}</UniqueIdentifier>
</Filter>
<Filter Include="algorithms\chameleon">
<UniqueIdentifier>{5AB95D49-4648-E712-EF66-FB0DDBD4F7B8}</UniqueIdentifier>
</Filter>
<Filter Include="algorithms\chameleon\core">
<UniqueIdentifier>{1252540B-7E14-8895-C7C4-162233C66073}</UniqueIdentifier>
</Filter>
<Filter Include="algorithms\chameleon\dictionary">
<UniqueIdentifier>{3F6054BC-AB43-63FC-B446-913820A9294D}</UniqueIdentifier>
</Filter>
<Filter Include="algorithms\cheetah">
<UniqueIdentifier>{C09C0CB6-AC80-CD0B-15E1-C75E01E4B78D}</UniqueIdentifier>
</Filter>
<Filter Include="algorithms\cheetah\core">
<UniqueIdentifier>{3845B705-A47C-FF9C-AD86-63C0193D2696}</UniqueIdentifier>
</Filter>
<Filter Include="algorithms\cheetah\dictionary">
<UniqueIdentifier>{E597066D-51B0-ED96-1A5D-7D3086348230}</UniqueIdentifier>
</Filter>
<Filter Include="algorithms\lion">
<UniqueIdentifier>{40FA2C44-AC85-9A08-B596-1DFD21A1F608}</UniqueIdentifier>
</Filter>
<Filter Include="algorithms\lion\core">
<UniqueIdentifier>{B8122E82-A4A1-B74B-4DC0-CB46392EC8F1}</UniqueIdentifier>
</Filter>
<Filter Include="algorithms\lion\dictionary">
<UniqueIdentifier>{6585C1C9-5195-6D9A-BA4E-D2B8A67D5C33}</UniqueIdentifier>
</Filter>
<Filter Include="algorithms\lion\forms">
<UniqueIdentifier>{B6B426C8-2221-E2C2-EB14-7A205740042B}</UniqueIdentifier>
</Filter>
<Filter Include="buffers">
<UniqueIdentifier>{B2DFE593-1EBF-642F-27D7-EF059335CB90}</UniqueIdentifier>
</Filter>
<Filter Include="structure">
<UniqueIdentifier>{367A73B3-A2E4-272A-EB22-D9CF57CC057F}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\src\algorithms\algorithms.h">
<Filter>algorithms</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\chameleon\chameleon.h">
<Filter>algorithms\chameleon</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\chameleon\core\chameleon_decode.h">
<Filter>algorithms\chameleon\core</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\chameleon\core\chameleon_encode.h">
<Filter>algorithms\chameleon\core</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\chameleon\dictionary\chameleon_dictionary.h">
<Filter>algorithms\chameleon\dictionary</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\cheetah\cheetah.h">
<Filter>algorithms\cheetah</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\cheetah\core\cheetah_decode.h">
<Filter>algorithms\cheetah\core</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\cheetah\core\cheetah_encode.h">
<Filter>algorithms\cheetah\core</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\cheetah\dictionary\cheetah_dictionary.h">
<Filter>algorithms\cheetah\dictionary</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\dictionaries.h">
<Filter>algorithms</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\lion\core\lion_decode.h">
<Filter>algorithms\lion\core</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\lion\core\lion_encode.h">
<Filter>algorithms\lion\core</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\lion\dictionary\lion_dictionary.h">
<Filter>algorithms\lion\dictionary</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\lion\forms\lion_form_model.h">
<Filter>algorithms\lion\forms</Filter>
</ClInclude>
<ClInclude Include="..\src\algorithms\lion\lion.h">
<Filter>algorithms\lion</Filter>
</ClInclude>
<ClInclude Include="..\src\buffers\buffer.h">
<Filter>buffers</Filter>
</ClInclude>
<ClInclude Include="..\src\density_api.h" />
<ClInclude Include="..\src\globals.h" />
<ClInclude Include="..\src\structure\header.h">
<Filter>structure</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\src\algorithms\algorithms.c">
<Filter>algorithms</Filter>
</ClCompile>
<ClCompile Include="..\src\algorithms\chameleon\core\chameleon_decode.c">
<Filter>algorithms\chameleon\core</Filter>
</ClCompile>
<ClCompile Include="..\src\algorithms\chameleon\core\chameleon_encode.c">
<Filter>algorithms\chameleon\core</Filter>
</ClCompile>
<ClCompile Include="..\src\algorithms\cheetah\core\cheetah_decode.c">
<Filter>algorithms\cheetah\core</Filter>
</ClCompile>
<ClCompile Include="..\src\algorithms\cheetah\core\cheetah_encode.c">
<Filter>algorithms\cheetah\core</Filter>
</ClCompile>
<ClCompile Include="..\src\algorithms\dictionaries.c">
<Filter>algorithms</Filter>
</ClCompile>
<ClCompile Include="..\src\algorithms\lion\core\lion_decode.c">
<Filter>algorithms\lion\core</Filter>
</ClCompile>
<ClCompile Include="..\src\algorithms\lion\core\lion_encode.c">
<Filter>algorithms\lion\core</Filter>
</ClCompile>
<ClCompile Include="..\src\algorithms\lion\forms\lion_form_model.c">
<Filter>algorithms\lion\forms</Filter>
</ClCompile>
<ClCompile Include="..\src\buffers\buffer.c">
<Filter>buffers</Filter>
</ClCompile>
<ClCompile Include="..\src\globals.c" />
<ClCompile Include="..\src\structure\header.c">
<Filter>structure</Filter>
</ClCompile>
</ItemGroup>
</Project>

43
contrib/density/src/algorithms/algorithms.c Normal file
View File

@ -0,0 +1,43 @@
/*
* Density
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 14/10/15 02:06
*/
#include "algorithms.h"
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_algorithms_prepare_state(density_algorithm_state *const DENSITY_RESTRICT state, void *const DENSITY_RESTRICT dictionary) {
state->dictionary = dictionary;
state->copy_penalty = 0;
state->copy_penalty_start = 1;
state->previous_incompressible = false;
state->counter = 0;
}

78
contrib/density/src/algorithms/algorithms.h Normal file
View File

@ -0,0 +1,78 @@
/*
* Density
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 3/02/15 19:51
*/
#ifndef DENSITY_ALGORITHMS_H
#define DENSITY_ALGORITHMS_H
#include "../globals.h"
typedef enum {
DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED = 0,
DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING,
DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL,
DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL
} density_algorithm_exit_status;
typedef struct {
void *dictionary;
uint_fast8_t copy_penalty;
uint_fast8_t copy_penalty_start;
bool previous_incompressible;
uint_fast64_t counter;
} density_algorithm_state;
#define DENSITY_ALGORITHM_COPY(work_block_size)\
DENSITY_MEMCPY(*out, *in, work_block_size);\
*in += work_block_size;\
*out += work_block_size;
#define DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START\
if(!(--state->copy_penalty))\
state->copy_penalty_start++;
#define DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START\
if (state->copy_penalty_start & ~0x1)\
state->copy_penalty_start >>= 1;
#define DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY(span, work_block_size)\
if (DENSITY_UNLIKELY(span & ~(work_block_size - 1))) {\
if (state->previous_incompressible)\
state->copy_penalty = state->copy_penalty_start;\
state->previous_incompressible = true;\
} else\
state->previous_incompressible = false;
DENSITY_WINDOWS_EXPORT void density_algorithms_prepare_state(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, void *const DENSITY_RESTRICT_DECLARE);
#endif

70
contrib/density/src/algorithms/chameleon/chameleon.h Normal file
View File

@ -0,0 +1,70 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 24/10/13 11:57
*
* -------------------
* Chameleon algorithm
* -------------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Hash based superfast kernel
*/
#ifndef DENSITY_CHAMELEON_H
#define DENSITY_CHAMELEON_H
#include "../../globals.h"
#define DENSITY_CHAMELEON_HASH_BITS 16
#define DENSITY_CHAMELEON_HASH_MULTIPLIER (uint32_t)0x9D6EF916lu
#define DENSITY_CHAMELEON_HASH_ALGORITHM(value32) (uint16_t)((value32 * DENSITY_CHAMELEON_HASH_MULTIPLIER) >> (32 - DENSITY_CHAMELEON_HASH_BITS))
typedef enum {
DENSITY_CHAMELEON_SIGNATURE_FLAG_CHUNK = 0x0,
DENSITY_CHAMELEON_SIGNATURE_FLAG_MAP = 0x1,
} DENSITY_CHAMELEON_SIGNATURE_FLAG;
typedef uint64_t density_chameleon_signature;
#define DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE (density_bitsizeof(density_chameleon_signature) * sizeof(uint32_t)) // Uncompressed chunks
#define DENSITY_CHAMELEON_DECOMPRESSED_BODY_SIZE_PER_SIGNATURE (density_bitsizeof(density_chameleon_signature) * sizeof(uint32_t))
#define DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE (sizeof(density_chameleon_signature) + DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE)
#define DENSITY_CHAMELEON_DECOMPRESSED_UNIT_SIZE (DENSITY_CHAMELEON_DECOMPRESSED_BODY_SIZE_PER_SIGNATURE)
#define DENSITY_CHAMELEON_WORK_BLOCK_SIZE 256
#endif

254
contrib/density/src/algorithms/chameleon/core/chameleon_decode.c Normal file
View File

@ -0,0 +1,254 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 23/06/15 22:11
*
* -------------------
* Chameleon algorithm
* -------------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Hash based superfast kernel
*/
#include "chameleon_decode.h"
DENSITY_FORCE_INLINE void density_chameleon_decode_process_compressed(const uint16_t hash, uint8_t **DENSITY_RESTRICT out, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) {
DENSITY_MEMCPY(*out, &dictionary->entries[hash].as_uint32_t, sizeof(uint32_t));
}
DENSITY_FORCE_INLINE void density_chameleon_decode_process_uncompressed(const uint32_t chunk, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) {
const uint16_t hash = DENSITY_CHAMELEON_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(chunk));
(&dictionary->entries[hash])->as_uint32_t = chunk; // Does not ensure dictionary content consistency between endiannesses
}
DENSITY_FORCE_INLINE void density_chameleon_decode_kernel(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const density_bool compressed, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) {
if (compressed) {
uint16_t hash;
DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t));
density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16(hash), out, dictionary);
*in += sizeof(uint16_t);
} else {
uint32_t unit;
DENSITY_MEMCPY(&unit, *in, sizeof(uint32_t));
density_chameleon_decode_process_uncompressed(unit, dictionary);
DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t));
*in += sizeof(uint32_t);
}
*out += sizeof(uint32_t);
}
DENSITY_FORCE_INLINE void density_chameleon_decode_kernel_dual(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const density_chameleon_signature signature, const uint_fast8_t shift, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) {
uint32_t var_32;
uint64_t var_64;
switch((signature >> shift) & 0x3) {
case 0x0:
DENSITY_MEMCPY(&var_64, *in, sizeof(uint32_t) + sizeof(uint32_t));
#ifdef DENSITY_LITTLE_ENDIAN
density_chameleon_decode_process_uncompressed((uint32_t)(var_64 & 0xffffffff), dictionary);
#endif
density_chameleon_decode_process_uncompressed((uint32_t)(var_64 >> density_bitsizeof(uint32_t)), dictionary);
#ifdef DENSITY_BIG_ENDIAN
density_chameleon_decode_process_uncompressed((uint32_t)(var_64 & 0xffffffff), dictionary);
#endif
DENSITY_MEMCPY(*out, &var_64, sizeof(uint32_t) + sizeof(uint32_t));
*in += (sizeof(uint32_t) + sizeof(uint32_t));
*out += sizeof(uint64_t);
break;
case 0x1:
DENSITY_MEMCPY(&var_64, *in, sizeof(uint16_t) + sizeof(uint32_t));
#ifdef DENSITY_LITTLE_ENDIAN
density_chameleon_decode_process_compressed((uint16_t)(var_64 & 0xffff), out, dictionary);
var_32 = (uint32_t)((var_64 >> density_bitsizeof(uint16_t)) & 0xffffffff);
density_chameleon_decode_process_uncompressed(var_32, dictionary);
DENSITY_MEMCPY(*out + sizeof(uint32_t), &var_32, sizeof(uint32_t));
*out += sizeof(uint64_t);
#elif defined(DENSITY_BIG_ENDIAN)
density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16((uint16_t)((var_64 >> (density_bitsizeof(uint16_t) + density_bitsizeof(uint32_t))) & 0xffff)), out, dictionary);
var_32 = (uint32_t)((var_64 >> density_bitsizeof(uint16_t)) & 0xffffffff);
density_chameleon_decode_process_uncompressed(var_32, dictionary);
DENSITY_MEMCPY(*out + sizeof(uint32_t), &var_32, sizeof(uint32_t));
*out += sizeof(uint64_t);
#else
#error
#endif
*in += (sizeof(uint16_t) + sizeof(uint32_t));
break;
case 0x2:
DENSITY_MEMCPY(&var_64, *in, sizeof(uint32_t) + sizeof(uint16_t));
#ifdef DENSITY_LITTLE_ENDIAN
var_32 = (uint32_t)(var_64 & 0xffffffff);
density_chameleon_decode_process_uncompressed(var_32, dictionary);
DENSITY_MEMCPY(*out, &var_32, sizeof(uint32_t));
*out += sizeof(uint32_t);
density_chameleon_decode_process_compressed((uint16_t)((var_64 >> density_bitsizeof(uint32_t)) & 0xffff), out, dictionary);
*out += sizeof(uint32_t);
#elif defined(DENSITY_BIG_ENDIAN)
var_32 = (uint32_t)((var_64 >> density_bitsizeof(uint32_t)) & 0xffffffff);
density_chameleon_decode_process_uncompressed(var_32, dictionary);
DENSITY_MEMCPY(*out, &var_32, sizeof(uint32_t));
*out += sizeof(uint32_t);
density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16((uint16_t)((var_64 >> density_bitsizeof(uint16_t)) & 0xffff)), out, dictionary);
*out += sizeof(uint32_t);
#else
#error
#endif
*in += (sizeof(uint32_t) + sizeof(uint16_t));
break;
case 0x3:
DENSITY_MEMCPY(&var_32, *in, sizeof(uint16_t) + sizeof(uint16_t));
#ifdef DENSITY_LITTLE_ENDIAN
density_chameleon_decode_process_compressed((uint16_t)(var_32 & 0xffff), out, dictionary);
*out += sizeof(uint32_t);
#endif
density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16((uint16_t)(var_32 >> density_bitsizeof(uint16_t))), out, dictionary);
*out += sizeof(uint32_t);
#ifdef DENSITY_BIG_ENDIAN
density_chameleon_decode_process_compressed(DENSITY_LITTLE_ENDIAN_16((uint16_t)(var_32 & 0xffff)), out, dictionary);
*out += sizeof(uint32_t);
#endif
*in += (sizeof(uint16_t) + sizeof(uint16_t));
break;
}
}
DENSITY_FORCE_INLINE bool density_chameleon_decode_test_compressed(const density_chameleon_signature signature, const uint_fast8_t shift) {
return (density_bool const) ((signature >> shift) & DENSITY_CHAMELEON_SIGNATURE_FLAG_MAP);
}
DENSITY_FORCE_INLINE void density_chameleon_decode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const density_chameleon_signature signature, const uint_fast8_t shift, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) {
density_chameleon_decode_kernel(in, out, density_chameleon_decode_test_compressed(signature, shift), dictionary);
}
DENSITY_FORCE_INLINE void density_chameleon_decode_256(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const density_chameleon_signature signature, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary) {
uint_fast8_t count_a = 0;
uint_fast8_t count_b = 0;
#if defined(__clang__) || defined(_MSC_VER)
do {
DENSITY_UNROLL_2(density_chameleon_decode_kernel_dual(in, out, signature, count_a, dictionary); count_a+= 2);
} while (++count_b & 0xf);
#else
do {
DENSITY_UNROLL_2(density_chameleon_decode_4(in, out, signature, count_a ++, dictionary));
} while (++count_b & 0x1f);
#endif
}
DENSITY_FORCE_INLINE void density_chameleon_decode_read_signature(const uint8_t **DENSITY_RESTRICT in, density_chameleon_signature *DENSITY_RESTRICT signature) {
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(signature, *in, sizeof(density_chameleon_signature));
#elif defined(DENSITY_BIG_ENDIAN)
density_chameleon_signature endian_signature;
DENSITY_MEMCPY(&endian_signature, *in, sizeof(density_chameleon_signature));
*signature = DENSITY_LITTLE_ENDIAN_64(endian_signature);
#else
#error
#endif
*in += sizeof(density_chameleon_signature);
}
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_chameleon_decode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) {
if (out_size < DENSITY_CHAMELEON_DECOMPRESSED_UNIT_SIZE)
return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL;
density_chameleon_signature signature;
uint_fast8_t shift;
uint_fast64_t remaining;
const uint8_t *start = *in;
if (in_size < DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE) {
goto read_signature;
}
const uint8_t *in_limit = *in + in_size - DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE;
uint8_t *out_limit = *out + out_size - DENSITY_CHAMELEON_DECOMPRESSED_UNIT_SIZE;
while (DENSITY_LIKELY(*in <= in_limit && *out <= out_limit)) {
if (DENSITY_UNLIKELY(!(state->counter & 0xf))) {
DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START;
}
state->counter++;
if (DENSITY_UNLIKELY(state->copy_penalty)) {
DENSITY_ALGORITHM_COPY(DENSITY_CHAMELEON_WORK_BLOCK_SIZE);
DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START;
} else {
const uint8_t *in_start = *in;
density_chameleon_decode_read_signature(in, &signature);
density_chameleon_decode_256(in, out, signature, (density_chameleon_dictionary *const) state->dictionary);
DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*in - in_start), DENSITY_CHAMELEON_WORK_BLOCK_SIZE);
}
}
if (*out > out_limit)
return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL;
read_signature:
if (in_size - (*in - start) < sizeof(density_chameleon_signature))
return DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL;
shift = 0;
density_chameleon_decode_read_signature(in, &signature);
read_and_decode_4:
switch (in_size - (*in - start)) {
case 0:
case 1:
if (density_chameleon_decode_test_compressed(signature, shift))
return DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING;
else // End marker
goto process_remaining_bytes;
case 2:
case 3:
if (density_chameleon_decode_test_compressed(signature, shift++))
density_chameleon_decode_kernel(in, out, true, (density_chameleon_dictionary *const) state->dictionary);
else // End marker
goto process_remaining_bytes;
break;
default:
density_chameleon_decode_4(in, out, signature, shift++, (density_chameleon_dictionary *const) state->dictionary);
break;
}
if (DENSITY_UNLIKELY(shift == density_bitsizeof(density_chameleon_signature)))
goto read_signature;
else
goto read_and_decode_4;
process_remaining_bytes:
remaining = in_size - (*in - start);
DENSITY_ALGORITHM_COPY(remaining);
return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED;
}

53
contrib/density/src/algorithms/chameleon/core/chameleon_decode.h Normal file
View File

@ -0,0 +1,53 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 23/06/15 21:49
*
* -------------------
* Chameleon algorithm
* -------------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Hash based superfast kernel
*/
#ifndef DENSITY_CHAMELEON_DECODE_H
#define DENSITY_CHAMELEON_DECODE_H
#include "../dictionary/chameleon_dictionary.h"
#include "../../algorithms.h"
DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_chameleon_decode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t);
#endif

179
contrib/density/src/algorithms/chameleon/core/chameleon_encode.c Normal file
View File

@ -0,0 +1,179 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 23/06/15 22:02
*
* -------------------
* Chameleon algorithm
* -------------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Hash based superfast kernel
*/
#include "chameleon_encode.h"
DENSITY_FORCE_INLINE void density_chameleon_encode_prepare_signature(uint8_t **DENSITY_RESTRICT out, density_chameleon_signature **DENSITY_RESTRICT signature_pointer, density_chameleon_signature *const DENSITY_RESTRICT signature) {
*signature = 0;
*signature_pointer = (density_chameleon_signature *) *out;
*out += sizeof(density_chameleon_signature);
}
DENSITY_FORCE_INLINE void density_chameleon_encode_kernel(uint8_t **DENSITY_RESTRICT out, const uint16_t hash, const uint_fast8_t shift, density_chameleon_signature *const DENSITY_RESTRICT signature, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) {
density_chameleon_dictionary_entry *const found = &dictionary->entries[hash];
switch (*unit ^ found->as_uint32_t) {
case 0:
*signature |= ((uint64_t) DENSITY_CHAMELEON_SIGNATURE_FLAG_MAP << shift);
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t));
#elif defined(DENSITY_BIG_ENDIAN)
const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash);
DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t));
#else
#error
#endif
*out += sizeof(uint16_t);
break;
default:
found->as_uint32_t = *unit; // Does not ensure dictionary content consistency between endiannesses
DENSITY_MEMCPY(*out, unit, sizeof(uint32_t));
*out += sizeof(uint32_t);
break;
}
}
DENSITY_FORCE_INLINE void density_chameleon_encode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, const uint_fast8_t shift, density_chameleon_signature *const DENSITY_RESTRICT signature, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) {
DENSITY_MEMCPY(unit, *in, sizeof(uint32_t));
density_chameleon_encode_kernel(out, DENSITY_CHAMELEON_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)), shift, signature, dictionary, unit);
*in += sizeof(uint32_t);
}
DENSITY_FORCE_INLINE void density_chameleon_encode_256(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, density_chameleon_signature *const DENSITY_RESTRICT signature, density_chameleon_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) {
uint_fast8_t count = 0;
#ifdef __clang__
for (uint_fast8_t count_b = 0; count_b < 32; count_b++) {
DENSITY_UNROLL_2(density_chameleon_encode_4(in, out, count++, signature, dictionary, unit));
}
#else
for (uint_fast8_t count_b = 0; count_b < 16; count_b++) {
DENSITY_UNROLL_4(density_chameleon_encode_4(in, out, count++, signature, dictionary, unit));
}
#endif
}
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_chameleon_encode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) {
if (out_size < DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE)
return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL;
density_chameleon_signature signature;
density_chameleon_signature *signature_pointer;
uint32_t unit;
uint8_t *out_limit = *out + out_size - DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE;
uint_fast64_t limit_256 = (in_size >> 8);
while (DENSITY_LIKELY(limit_256-- && *out <= out_limit)) {
if (DENSITY_UNLIKELY(!(state->counter & 0xf))) {
DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START;
}
state->counter++;
if (DENSITY_UNLIKELY(state->copy_penalty)) {
DENSITY_ALGORITHM_COPY(DENSITY_CHAMELEON_WORK_BLOCK_SIZE);
DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START;
} else {
const uint8_t *out_start = *out;
density_chameleon_encode_prepare_signature(out, &signature_pointer, &signature);
DENSITY_PREFETCH(*in + DENSITY_CHAMELEON_WORK_BLOCK_SIZE);
density_chameleon_encode_256(in, out, &signature, (density_chameleon_dictionary *const) state->dictionary, &unit);
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_chameleon_signature));
#elif defined(DENSITY_BIG_ENDIAN)
const density_chameleon_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature);
DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_chameleon_signature));
#else
#error
#endif
DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*out - out_start), DENSITY_CHAMELEON_WORK_BLOCK_SIZE);
}
}
if (*out > out_limit)
return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL;
uint_fast64_t remaining;
switch (in_size & 0xff) {
case 0:
case 1:
case 2:
case 3:
density_chameleon_encode_prepare_signature(out, &signature_pointer, &signature);
signature = ((uint64_t) DENSITY_CHAMELEON_SIGNATURE_FLAG_CHUNK); // End marker
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_chameleon_signature));
#elif defined(DENSITY_BIG_ENDIAN)
const density_chameleon_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature);
DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_chameleon_signature));
#else
#error
#endif
goto process_remaining_bytes;
default:
break;
}
const uint_fast64_t limit_4 = (in_size & 0xff) >> 2;
density_chameleon_encode_prepare_signature(out, &signature_pointer, &signature);
for (uint_fast8_t shift = 0; shift != limit_4; shift++)
density_chameleon_encode_4(in, out, shift, &signature, (density_chameleon_dictionary *const) state->dictionary, &unit);
signature |= ((uint64_t) DENSITY_CHAMELEON_SIGNATURE_FLAG_CHUNK << limit_4); // End marker
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_chameleon_signature));
#elif defined(DENSITY_BIG_ENDIAN)
const density_chameleon_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature);
DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_chameleon_signature));
#else
#error
#endif
process_remaining_bytes:
remaining = in_size & 0x3;
if (remaining)
DENSITY_ALGORITHM_COPY(remaining);
return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED;
}

53
contrib/density/src/algorithms/chameleon/core/chameleon_encode.h Normal file
View File

@ -0,0 +1,53 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 23/06/15 21:51
*
* -------------------
* Chameleon algorithm
* -------------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Hash based superfast kernel
*/
#ifndef DENSITY_CHAMELEON_ENCODE_H
#define DENSITY_CHAMELEON_ENCODE_H
#include "../dictionary/chameleon_dictionary.h"
#include "../../algorithms.h"
DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_chameleon_encode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t);
#endif

63
contrib/density/src/algorithms/chameleon/dictionary/chameleon_dictionary.h Normal file
View File

@ -0,0 +1,63 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 24/10/13 12:05
*
* -------------------
* Chameleon algorithm
* -------------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Hash based superfast kernel
*/
#ifndef DENSITY_CHAMELEON_DICTIONARY_H
#define DENSITY_CHAMELEON_DICTIONARY_H
#include "../chameleon.h"
#include <string.h>
#pragma pack(push)
#pragma pack(4)
typedef struct {
uint32_t as_uint32_t;
} density_chameleon_dictionary_entry;
typedef struct {
density_chameleon_dictionary_entry entries[1 << DENSITY_CHAMELEON_HASH_BITS];
} density_chameleon_dictionary;
#pragma pack(pop)
#endif

73
contrib/density/src/algorithms/cheetah/cheetah.h Normal file
View File

@ -0,0 +1,73 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 06/12/13 20:10
*
* -----------------
* Cheetah algorithm
* -----------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
* Piotr Tarsa (https://github.com/tarsa)
*
* Description
* Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup
*/
#ifndef DENSITY_CHEETAH_H
#define DENSITY_CHEETAH_H
#include "../../globals.h"
#define DENSITY_CHEETAH_HASH_BITS 16
#define DENSITY_CHEETAH_HASH_MULTIPLIER (uint32_t)0x9D6EF916lu
#define DENSITY_CHEETAH_HASH_ALGORITHM(value32) (uint16_t)(value32 * DENSITY_CHEETAH_HASH_MULTIPLIER >> (32 - DENSITY_CHEETAH_HASH_BITS))
typedef enum {
DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED = 0x0,
DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_A = 0x1,
DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_B = 0x2,
DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK = 0x3,
} DENSITY_CHEETAH_SIGNATURE_FLAG;
typedef uint64_t density_cheetah_signature;
#define DENSITY_CHEETAH_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE ((density_bitsizeof(density_cheetah_signature) >> 1) * sizeof(uint32_t)) // Uncompressed chunks
#define DENSITY_CHEETAH_DECOMPRESSED_BODY_SIZE_PER_SIGNATURE ((density_bitsizeof(density_cheetah_signature) >> 1) * sizeof(uint32_t))
#define DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE (sizeof(density_cheetah_signature) + DENSITY_CHEETAH_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE)
#define DENSITY_CHEETAH_DECOMPRESSED_UNIT_SIZE (DENSITY_CHEETAH_DECOMPRESSED_BODY_SIZE_PER_SIGNATURE)
#define DENSITY_CHEETAH_WORK_BLOCK_SIZE 128
#endif

266
contrib/density/src/algorithms/cheetah/core/cheetah_decode.c Normal file
View File

@ -0,0 +1,266 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 24/06/15 0:32
*
* -----------------
* Cheetah algorithm
* -----------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
* Piotr Tarsa (https://github.com/tarsa)
*
* Description
* Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup
*/
#include "cheetah_decode.h"
DENSITY_FORCE_INLINE void density_cheetah_decode_process_predicted(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) {
const uint32_t unit = dictionary->prediction_entries[*last_hash].next_chunk_prediction;
DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t));
*last_hash = DENSITY_CHEETAH_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(unit));
}
DENSITY_FORCE_INLINE void density_cheetah_decode_process_compressed_a(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, const uint16_t hash) {
DENSITY_PREFETCH(&dictionary->prediction_entries[hash]);
const uint32_t unit = dictionary->entries[hash].chunk_a;
DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t));
dictionary->prediction_entries[*last_hash].next_chunk_prediction = unit;
*last_hash = hash;
}
DENSITY_FORCE_INLINE void density_cheetah_decode_process_compressed_b(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, const uint16_t hash) {
DENSITY_PREFETCH(&dictionary->prediction_entries[hash]);
density_cheetah_dictionary_entry *const entry = &dictionary->entries[hash];
const uint32_t unit = entry->chunk_b;
entry->chunk_b = entry->chunk_a;
entry->chunk_a = unit; // Does not ensure dictionary content consistency between endiannesses
DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t));
dictionary->prediction_entries[*last_hash].next_chunk_prediction = unit;
*last_hash = hash;
}
DENSITY_FORCE_INLINE void density_cheetah_decode_process_uncompressed(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, const uint32_t unit) {
const uint16_t hash = DENSITY_CHEETAH_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(unit));
DENSITY_PREFETCH(&dictionary->prediction_entries[hash]);
density_cheetah_dictionary_entry *const entry = &dictionary->entries[hash];
entry->chunk_b = entry->chunk_a;
entry->chunk_a = unit; // Does not ensure dictionary content consistency between endiannesses
DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t));
dictionary->prediction_entries[*last_hash].next_chunk_prediction = unit; // Does not ensure dictionary content consistency between endiannesses
*last_hash = hash;
}
DENSITY_FORCE_INLINE void density_cheetah_decode_kernel_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const uint8_t flag, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) {
uint16_t hash;
uint32_t unit;
switch (flag) {
case DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED:
density_cheetah_decode_process_predicted(out, last_hash, dictionary);
break;
case DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_A:
DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t));
density_cheetah_decode_process_compressed_a(out, last_hash, dictionary, DENSITY_LITTLE_ENDIAN_16(hash));
*in += sizeof(uint16_t);
break;
case DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_B:
DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t));
density_cheetah_decode_process_compressed_b(out, last_hash, dictionary, DENSITY_LITTLE_ENDIAN_16(hash));
*in += sizeof(uint16_t);
break;
default: // DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK
DENSITY_MEMCPY(&unit, *in, sizeof(uint32_t));
density_cheetah_decode_process_uncompressed(out, last_hash, dictionary, unit);
*in += sizeof(uint32_t);
break;
}
*out += sizeof(uint32_t);
}
DENSITY_FORCE_INLINE void density_cheetah_decode_kernel_16(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const uint8_t flags, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) {
uint16_t hash;
uint32_t unit;
switch (flags) {
DENSITY_CASE_GENERATOR_4_4_COMBINED(\
density_cheetah_decode_process_predicted(out, last_hash, dictionary);, \
DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED, \
DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t)); \
density_cheetah_decode_process_compressed_a(out, last_hash, dictionary, DENSITY_LITTLE_ENDIAN_16(hash));\
*in += sizeof(uint16_t);, \
DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_A, \
DENSITY_MEMCPY(&hash, *in, sizeof(uint16_t)); \
density_cheetah_decode_process_compressed_b(out, last_hash, dictionary, DENSITY_LITTLE_ENDIAN_16(hash));\
*in += sizeof(uint16_t);, \
DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_B, \
DENSITY_MEMCPY(&unit, *in, sizeof(uint32_t)); \
density_cheetah_decode_process_uncompressed(out, last_hash, dictionary, unit);\
*in += sizeof(uint32_t);, \
DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK, \
*out += sizeof(uint32_t);, \
2\
);
default:
break;
}
*out += sizeof(uint32_t);
}
DENSITY_FORCE_INLINE uint8_t density_cheetah_decode_read_flag(const density_cheetah_signature signature, const uint_fast8_t shift) {
return (uint8_t const) ((signature >> shift) & 0x3);
}
DENSITY_FORCE_INLINE void density_cheetah_decode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const density_cheetah_signature signature, const uint_fast8_t shift, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) {
density_cheetah_decode_kernel_4(in, out, last_hash, density_cheetah_decode_read_flag(signature, shift), dictionary);
}
DENSITY_FORCE_INLINE void density_cheetah_decode_16(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const density_cheetah_signature signature, const uint_fast8_t shift, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) {
density_cheetah_decode_kernel_16(in, out, last_hash, (uint8_t const) ((signature >> shift) & 0xff), dictionary);
}
DENSITY_FORCE_INLINE void density_cheetah_decode_128(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const density_cheetah_signature signature, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary) {
#ifdef __clang__
uint_fast8_t count = 0;
for (uint_fast8_t count_b = 0; count_b < 8; count_b ++) {
density_cheetah_decode_16(in, out, last_hash, signature, count, dictionary);
count += 8;
}
#else
for (uint_fast8_t count_b = 0; count_b < density_bitsizeof(density_cheetah_signature); count_b += 8)
density_cheetah_decode_16(in, out, last_hash, signature, count_b, dictionary);
#endif
}
DENSITY_FORCE_INLINE void density_cheetah_decode_read_signature(const uint8_t **DENSITY_RESTRICT in, density_cheetah_signature *DENSITY_RESTRICT signature) {
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(signature, *in, sizeof(density_cheetah_signature));
#elif defined(DENSITY_BIG_ENDIAN)
density_cheetah_signature endian_signature;
DENSITY_MEMCPY(&endian_signature, *in, sizeof(density_cheetah_signature));
*signature = DENSITY_LITTLE_ENDIAN_64(endian_signature);
#else
#error
#endif
*in += sizeof(density_cheetah_signature);
}
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_cheetah_decode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) {
if (out_size < DENSITY_CHEETAH_DECOMPRESSED_UNIT_SIZE)
return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL;
density_cheetah_signature signature;
uint_fast8_t shift;
uint_fast64_t remaining;
uint_fast16_t last_hash = 0;
uint8_t flag;
const uint8_t *start = *in;
if (in_size < DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE) {
goto read_signature;
}
const uint8_t *in_limit = *in + in_size - DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE;
uint8_t *out_limit = *out + out_size - DENSITY_CHEETAH_DECOMPRESSED_UNIT_SIZE;
while (DENSITY_LIKELY(*in <= in_limit && *out <= out_limit)) {
if (DENSITY_UNLIKELY(!(state->counter & 0x1f))) {
DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START;
}
state->counter++;
if (DENSITY_UNLIKELY(state->copy_penalty)) {
DENSITY_ALGORITHM_COPY(DENSITY_CHEETAH_WORK_BLOCK_SIZE);
DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START;
} else {
const uint8_t *in_start = *in;
density_cheetah_decode_read_signature(in, &signature);
density_cheetah_decode_128(in, out, &last_hash, signature, (density_cheetah_dictionary *const) state->dictionary);
DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*in - in_start), DENSITY_CHEETAH_WORK_BLOCK_SIZE);
}
}
if (*out > out_limit)
return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL;
read_signature:
if (in_size - (*in - start) < sizeof(density_cheetah_signature))
return DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL;
shift = 0;
density_cheetah_decode_read_signature(in, &signature);
read_and_decode_4:
switch (in_size - (*in - start)) {
case 0:
case 1:
switch (density_cheetah_decode_read_flag(signature, shift)) {
case DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK:
goto process_remaining_bytes; // End marker
case DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED:
density_cheetah_decode_kernel_4(in, out, &last_hash, DENSITY_CHEETAH_SIGNATURE_FLAG_PREDICTED, (density_cheetah_dictionary *const) state->dictionary);
shift += 2;
break;
default:
return DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING;
}
break;
case 2:
case 3:
flag = density_cheetah_decode_read_flag(signature, shift);
switch (flag) {
case DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK:
goto process_remaining_bytes; // End marker
default:
density_cheetah_decode_kernel_4(in, out, &last_hash, flag, (density_cheetah_dictionary *const) state->dictionary);
shift += 2;
break;
}
break;
default:
density_cheetah_decode_4(in, out, &last_hash, signature, shift, (density_cheetah_dictionary *const) state->dictionary);
shift += 2;
break;
}
if (DENSITY_UNLIKELY(shift == density_bitsizeof(density_cheetah_signature)))
goto read_signature;
else
goto read_and_decode_4;
process_remaining_bytes:
remaining = in_size - (*in - start);
DENSITY_ALGORITHM_COPY(remaining);
return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED;
}

54
contrib/density/src/algorithms/cheetah/core/cheetah_decode.h Normal file
View File

@ -0,0 +1,54 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 24/06/15 0:31
*
* -----------------
* Cheetah algorithm
* -----------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
* Piotr Tarsa (https://github.com/tarsa)
*
* Description
* Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup
*/
#ifndef DENSITY_CHEETAH_DECODE_H
#define DENSITY_CHEETAH_DECODE_H
#include "../dictionary/cheetah_dictionary.h"
#include "../../algorithms.h"
DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_cheetah_decode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t);
#endif

202
contrib/density/src/algorithms/cheetah/core/cheetah_encode.c Normal file
View File

@ -0,0 +1,202 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 23/06/15 23:29
*
* -----------------
* Cheetah algorithm
* -----------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
* Piotr Tarsa (https://github.com/tarsa)
*
* Description
* Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup
*/
#include "cheetah_encode.h"
DENSITY_FORCE_INLINE void density_cheetah_encode_prepare_signature(uint8_t **DENSITY_RESTRICT out, density_cheetah_signature **DENSITY_RESTRICT signature_pointer, density_cheetah_signature *const DENSITY_RESTRICT signature) {
*signature = 0;
*signature_pointer = (density_cheetah_signature *) *out;
*out += sizeof(density_cheetah_signature);
}
DENSITY_FORCE_INLINE void density_cheetah_encode_kernel(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const uint_fast16_t hash, const uint_fast8_t shift, density_cheetah_signature *const DENSITY_RESTRICT signature, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) {
uint32_t *predictedChunk = (uint32_t*) &dictionary->prediction_entries[*last_hash];
if (*predictedChunk ^ *unit) {
density_cheetah_dictionary_entry *found = &dictionary->entries[hash];
uint32_t *found_a = &found->chunk_a;
if (*found_a ^ *unit) {
uint32_t *found_b = &found->chunk_b;
if (*found_b ^ *unit) {
*signature |= ((uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK << shift);
DENSITY_MEMCPY(*out, unit, sizeof(uint32_t));
*out += sizeof(uint32_t);
} else {
*signature |= ((uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_B << shift);
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t));
#elif defined(DENSITY_BIG_ENDIAN)
const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash);
DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t));
#else
#error
#endif
*out += sizeof(uint16_t);
}
*found_b = *found_a;
*found_a = *unit; // Does not ensure dictionary content consistency between endiannesses
} else {
*signature |= ((uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_MAP_A << shift);
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t));
#elif defined(DENSITY_BIG_ENDIAN)
const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash);
DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t));
#else
#error
#endif
*out += sizeof(uint16_t);
}
*predictedChunk = *unit; // Does not ensure dictionary content consistency between endiannesses
}
*last_hash = hash;
}
DENSITY_FORCE_INLINE void density_cheetah_encode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, const uint_fast8_t shift, density_cheetah_signature *const DENSITY_RESTRICT signature, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) {
DENSITY_MEMCPY(unit, *in, sizeof(uint32_t));
*in += sizeof(uint32_t);
density_cheetah_encode_kernel(out, last_hash, DENSITY_CHEETAH_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)), shift, signature, dictionary, unit);
}
DENSITY_FORCE_INLINE void density_cheetah_encode_128(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_cheetah_signature *const DENSITY_RESTRICT signature, density_cheetah_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT unit) {
uint_fast8_t count = 0;
#ifdef __clang__
for(; count < density_bitsizeof(density_cheetah_signature); count += 2) {
density_cheetah_encode_4(in, out, last_hash, count, signature, dictionary, unit);
}
#else
for (uint_fast8_t count_b = 0; count_b < 16; count_b++) {
DENSITY_UNROLL_2(\
density_cheetah_encode_4(in, out, last_hash, count, signature, dictionary, unit);\
count += 2);
}
#endif
}
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_cheetah_encode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) {
if (out_size < DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE)
return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL;
density_cheetah_signature signature;
density_cheetah_signature *signature_pointer;
uint_fast16_t last_hash = 0;
uint32_t unit;
uint8_t *out_limit = *out + out_size - DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE;
uint_fast64_t limit_128 = (in_size >> 7);
while (DENSITY_LIKELY(limit_128-- && *out <= out_limit)) {
if (DENSITY_UNLIKELY(!(state->counter & 0x1f))) {
DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START;
}
state->counter++;
if (DENSITY_UNLIKELY(state->copy_penalty)) {
DENSITY_ALGORITHM_COPY(DENSITY_CHEETAH_WORK_BLOCK_SIZE);
DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START;
} else {
const uint8_t *out_start = *out;
density_cheetah_encode_prepare_signature(out, &signature_pointer, &signature);
DENSITY_PREFETCH(*in + DENSITY_CHEETAH_WORK_BLOCK_SIZE);
density_cheetah_encode_128(in, out, &last_hash, &signature, (density_cheetah_dictionary *const) state->dictionary, &unit);
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_cheetah_signature));
#elif defined(DENSITY_BIG_ENDIAN)
const density_cheetah_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature);
DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_cheetah_signature));
#else
#error
#endif
DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*out - out_start), DENSITY_CHEETAH_WORK_BLOCK_SIZE);
}
}
if (*out > out_limit)
return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL;
uint_fast64_t remaining;
switch (in_size & 0x7f) {
case 0:
case 1:
case 2:
case 3:
density_cheetah_encode_prepare_signature(out, &signature_pointer, &signature);
signature = (uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK; // End marker
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_cheetah_signature));
#elif defined(DENSITY_BIG_ENDIAN)
const density_cheetah_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature);
DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_cheetah_signature));
#else
#error
#endif
goto process_remaining_bytes;
default:
break;
}
const uint_fast64_t limit_4 = ((in_size & 0x7f) >> 2) << 1; // 4-byte units times number of signature flag bits
density_cheetah_encode_prepare_signature(out, &signature_pointer, &signature);
for (uint_fast8_t shift = 0; shift != limit_4; shift += 2)
density_cheetah_encode_4(in, out, &last_hash, shift, &signature, (density_cheetah_dictionary *const) state->dictionary, &unit);
signature |= ((uint64_t) DENSITY_CHEETAH_SIGNATURE_FLAG_CHUNK << limit_4); // End marker
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_cheetah_signature));
#elif defined(DENSITY_BIG_ENDIAN)
const density_cheetah_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature);
DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_cheetah_signature));
#else
#error
#endif
process_remaining_bytes:
remaining = in_size & 0x3;
if (remaining)
DENSITY_ALGORITHM_COPY(remaining);
return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED;
}

54
contrib/density/src/algorithms/cheetah/core/cheetah_encode.h Normal file
View File

@ -0,0 +1,54 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 23/06/15 23:29
*
* -----------------
* Cheetah algorithm
* -----------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
* Piotr Tarsa (https://github.com/tarsa)
*
* Description
* Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup
*/
#ifndef DENSITY_CHEETAH_ENCODE_H
#define DENSITY_CHEETAH_ENCODE_H
#include "../dictionary/cheetah_dictionary.h"
#include "../../algorithms.h"
DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_cheetah_encode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t);
#endif

70
contrib/density/src/algorithms/cheetah/dictionary/cheetah_dictionary.h Normal file
View File

@ -0,0 +1,70 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 06/12/13 20:20
*
* -----------------
* Cheetah algorithm
* -----------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
* Piotr Tarsa (https://github.com/tarsa)
*
* Description
* Very fast two level dictionary hash algorithm derived from Chameleon, with predictions lookup
*/
#ifndef DENSITY_CHEETAH_DICTIONARY_H
#define DENSITY_CHEETAH_DICTIONARY_H
#include "../cheetah.h"
#include <string.h>
#pragma pack(push)
#pragma pack(4)
typedef struct {
uint32_t chunk_a;
uint32_t chunk_b;
} density_cheetah_dictionary_entry;
typedef struct {
uint32_t next_chunk_prediction;
} density_cheetah_dictionary_prediction_entry;
typedef struct {
density_cheetah_dictionary_entry entries[1 << DENSITY_CHEETAH_HASH_BITS];
density_cheetah_dictionary_prediction_entry prediction_entries[1 << DENSITY_CHEETAH_HASH_BITS];
} density_cheetah_dictionary;
#pragma pack(pop)
#endif

48
contrib/density/src/algorithms/dictionaries.c Normal file
View File

@ -0,0 +1,48 @@
/*
* Density
*
* Copyright (c) 2018, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 13/01/18 17:31
*/
#include "dictionaries.h"
DENSITY_WINDOWS_EXPORT size_t density_get_dictionary_size(DENSITY_ALGORITHM algorithm) {
switch(algorithm) {
case DENSITY_ALGORITHM_CHAMELEON:
return sizeof(density_chameleon_dictionary);
case DENSITY_ALGORITHM_CHEETAH:
return sizeof(density_cheetah_dictionary);
case DENSITY_ALGORITHM_LION:
return sizeof(density_lion_dictionary);
default:
return 0;
}
}

45
contrib/density/src/algorithms/dictionaries.h Normal file
View File

@ -0,0 +1,45 @@
/*
* Density
*
* Copyright (c) 2018, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 13/01/18 17:26
*/
#ifndef DENSITY_DICTIONARIES_H
#define DENSITY_DICTIONARIES_H
#include "../globals.h"
#include "../algorithms/chameleon/dictionary/chameleon_dictionary.h"
#include "../algorithms/cheetah/dictionary/cheetah_dictionary.h"
#include "../algorithms/lion/dictionary/lion_dictionary.h"
DENSITY_WINDOWS_EXPORT size_t density_get_dictionary_size(DENSITY_ALGORITHM);
#endif

327
contrib/density/src/algorithms/lion/core/lion_decode.c Normal file
View File

@ -0,0 +1,327 @@
/*
* Density
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 24/06/15 20:55
*
* --------------
* Lion algorithm
* --------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Multiform compression algorithm
*/
#include "lion_decode.h"
DENSITY_FORCE_INLINE void density_lion_decode_read_signature(const uint8_t **DENSITY_RESTRICT in, uint_fast64_t *const DENSITY_RESTRICT signature) {
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(signature, *in, sizeof(density_lion_signature));
#elif defined(DENSITY_BIG_ENDIAN)
density_lion_signature endian_signature;
DENSITY_MEMCPY(&endian_signature, *in, sizeof(density_lion_signature));
*signature = DENSITY_LITTLE_ENDIAN_64(endian_signature);
#else
#error
#endif
*in += sizeof(density_lion_signature);
}
DENSITY_FORCE_INLINE void density_lion_decode_update_predictions_model(density_lion_dictionary_chunk_prediction_entry *const DENSITY_RESTRICT predictions, const uint32_t chunk) {
DENSITY_MEMMOVE((uint32_t *) predictions + 1, predictions, 2 * sizeof(uint32_t));
*(uint32_t *) predictions = chunk; // Move chunk to the top of the predictions list, does not ensure dictionary content consistency between endiannesses
}
DENSITY_FORCE_INLINE void density_lion_decode_update_dictionary_model(density_lion_dictionary_chunk_entry *const DENSITY_RESTRICT entry, const uint32_t chunk) {
DENSITY_MEMMOVE((uint32_t *) entry + 1, entry, 3 * sizeof(uint32_t));
*(uint32_t *) entry = chunk; // Does not ensure dictionary content consistency between endiannesses
}
DENSITY_FORCE_INLINE void density_lion_decode_read_hash(const uint8_t **DENSITY_RESTRICT in, uint16_t *DENSITY_RESTRICT const hash) {
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(hash, *in, sizeof(uint16_t));
#elif defined(DENSITY_BIG_ENDIAN)
uint16_t endian_hash;
DENSITY_MEMCPY(&endian_hash, *in, sizeof(uint16_t));
*hash = DENSITY_LITTLE_ENDIAN_16(endian_hash);
#else
#error
#endif
*in += sizeof(uint16_t);
}
DENSITY_FORCE_INLINE void density_lion_decode_prediction_generic(uint8_t **DENSITY_RESTRICT out, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) {
*hash = DENSITY_LION_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit));
DENSITY_MEMCPY(*out, unit, sizeof(uint32_t));
*out += sizeof(uint32_t);
}
DENSITY_FORCE_INLINE void density_lion_decode_dictionary_generic(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint32_t *DENSITY_RESTRICT const unit) {
DENSITY_MEMCPY(*out, unit, sizeof(uint32_t));
*out += sizeof(uint32_t);
density_lion_dictionary_chunk_prediction_entry *prediction = &(dictionary->predictions[*last_hash]);
density_lion_decode_update_predictions_model(prediction, *unit);
}
void density_lion_decode_prediction_a(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) {
(void)in;
*unit = dictionary->predictions[*last_hash].next_chunk_a;
density_lion_decode_prediction_generic(out, hash, unit);
*last_hash = *hash;
}
void density_lion_decode_prediction_b(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) {
(void)in;
density_lion_dictionary_chunk_prediction_entry *const prediction = &dictionary->predictions[*last_hash];
*unit = prediction->next_chunk_b;
density_lion_decode_update_predictions_model(prediction, *unit);
density_lion_decode_prediction_generic(out, hash, unit);
*last_hash = *hash;
}
void density_lion_decode_prediction_c(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) {
(void)in;
density_lion_dictionary_chunk_prediction_entry *const prediction = &dictionary->predictions[*last_hash];
*unit = prediction->next_chunk_c;
density_lion_decode_update_predictions_model(prediction, *unit);
density_lion_decode_prediction_generic(out, hash, unit);
*last_hash = *hash;
}
void density_lion_decode_dictionary_a(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) {
density_lion_decode_read_hash(in, hash);
DENSITY_PREFETCH(&dictionary->predictions[*hash]);
*unit = dictionary->chunks[*hash].chunk_a;
density_lion_decode_dictionary_generic(out, last_hash, dictionary, unit);
*last_hash = *hash;
}
void density_lion_decode_dictionary_b(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) {
density_lion_decode_read_hash(in, hash);
DENSITY_PREFETCH(&dictionary->predictions[*hash]);
density_lion_dictionary_chunk_entry *entry = &dictionary->chunks[*hash];
*unit = entry->chunk_b;
density_lion_decode_update_dictionary_model(entry, *unit);
density_lion_decode_dictionary_generic(out, last_hash, dictionary, unit);
*last_hash = *hash;
}
void density_lion_decode_dictionary_c(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) {
density_lion_decode_read_hash(in, hash);
DENSITY_PREFETCH(&dictionary->predictions[*hash]);
density_lion_dictionary_chunk_entry *entry = &dictionary->chunks[*hash];
*unit = entry->chunk_c;
density_lion_decode_update_dictionary_model(entry, *unit);
density_lion_decode_dictionary_generic(out, last_hash, dictionary, unit);
*last_hash = *hash;
}
void density_lion_decode_dictionary_d(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) {
density_lion_decode_read_hash(in, hash);
DENSITY_PREFETCH(&dictionary->predictions[*hash]);
density_lion_dictionary_chunk_entry *entry = &dictionary->chunks[*hash];
*unit = entry->chunk_d;
density_lion_decode_update_dictionary_model(entry, *unit);
density_lion_decode_dictionary_generic(out, last_hash, dictionary, unit);
*last_hash = *hash;
}
void density_lion_decode_plain(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, uint16_t *DENSITY_RESTRICT const hash, uint32_t *DENSITY_RESTRICT const unit) {
DENSITY_MEMCPY(unit, *in, sizeof(uint32_t));
*in += sizeof(uint32_t);
*hash = DENSITY_LION_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit));
density_lion_dictionary_chunk_entry *entry = &dictionary->chunks[*hash];
density_lion_decode_update_dictionary_model(entry, *unit);
DENSITY_MEMCPY(*out, unit, sizeof(uint32_t));
*out += sizeof(uint32_t);
density_lion_dictionary_chunk_prediction_entry *prediction = &(dictionary->predictions[*last_hash]);
density_lion_decode_update_predictions_model(prediction, *unit);
*last_hash = *hash;
}
DENSITY_FORCE_INLINE void density_lion_decode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const data, const DENSITY_LION_FORM form) {
uint16_t hash;
uint32_t unit;
data->attachments[form](in, out, last_hash, dictionary, &hash, &unit);
}
DENSITY_FORCE_INLINE DENSITY_LION_FORM density_lion_decode_read_form(const uint8_t **DENSITY_RESTRICT in, uint_fast64_t *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_form_data *const form_data) {
const uint_fast8_t trailing_zeroes = DENSITY_CTZ(0x80 | (*signature >> *shift));
if (DENSITY_LIKELY(!trailing_zeroes)) {
*shift = (uint_fast8_t)((*shift + 1) & 0x3f);
return density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool);
} else if (DENSITY_LIKELY(trailing_zeroes <= 6)) {
*shift = (uint_fast8_t)((*shift + (trailing_zeroes + 1)) & 0x3f);
return density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool + trailing_zeroes);
} else {
if (DENSITY_LIKELY(*shift <= (density_bitsizeof(density_lion_signature) - 7))) {
*shift = (uint_fast8_t)((*shift + 7) & 0x3f);
return density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool + 7);
} else {
density_lion_decode_read_signature(in, signature);
const uint_fast8_t primary_trailing_zeroes = (uint_fast8_t)(density_bitsizeof(density_lion_signature) - *shift);
const uint_fast8_t ctz_barrier_shift = (uint_fast8_t)(7 - primary_trailing_zeroes);
const uint_fast8_t secondary_trailing_zeroes = DENSITY_CTZ(((uint64_t)1 << ctz_barrier_shift) | *signature);
if (DENSITY_LIKELY(secondary_trailing_zeroes != ctz_barrier_shift))
*shift = (uint_fast8_t)(secondary_trailing_zeroes + 1);
else
*shift = secondary_trailing_zeroes;
return density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool + primary_trailing_zeroes + secondary_trailing_zeroes);
}
}
}
DENSITY_FORCE_INLINE void density_lion_decode_process_form(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const form_data, uint_fast64_t *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift) {
if (DENSITY_UNLIKELY(!*shift))
density_lion_decode_read_signature(in, signature);
switch ((*signature >> *shift) & 0x1) {
case 0:
density_lion_decode_4(in, out, last_hash, dictionary, form_data, density_lion_decode_read_form(in, signature, shift, form_data));
break;
default:
density_lion_decode_4(in, out, last_hash, dictionary, form_data, density_lion_form_model_increment_usage(form_data, (density_lion_form_node *) form_data->formsPool));
*shift = (uint_fast8_t)((*shift + 1) & 0x3f);
break;
}
}
DENSITY_FORCE_INLINE void density_lion_decode_256(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const form_data, uint_fast64_t *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift) {
#ifdef __clang__
for (uint_fast8_t count = 0; count < (DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG >> 2); count++) {
DENSITY_UNROLL_4(density_lion_decode_process_form(in, out, last_hash, dictionary, form_data, signature, shift));
}
#else
for (uint_fast8_t count = 0; count < (DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG >> 2); count++) {
DENSITY_UNROLL_4(density_lion_decode_process_form(in, out, last_hash, dictionary, form_data, signature, shift));
}
#endif
}
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_lion_decode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) {
if (out_size < DENSITY_LION_MAXIMUM_DECOMPRESSED_UNIT_SIZE)
return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL;
density_lion_signature signature = 0;
density_lion_form_data data;
density_lion_form_model_init(&data);
void (*attachments[DENSITY_LION_NUMBER_OF_FORMS])(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const) = {(void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_prediction_a, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_prediction_b, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_prediction_c, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_dictionary_a, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_dictionary_b, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_dictionary_c, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_dictionary_d, (void (*)(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) &density_lion_decode_plain};
density_lion_form_model_attach(&data, attachments);
uint_fast8_t shift = 0;
uint_fast64_t remaining;
uint_fast16_t last_hash = 0;
DENSITY_LION_FORM form;
const uint8_t *start = *in;
if (in_size < DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE) {
goto read_and_decode_4;
}
const uint8_t *in_limit = *in + in_size - DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE;
uint8_t *out_limit = *out + out_size - DENSITY_LION_MAXIMUM_DECOMPRESSED_UNIT_SIZE;
while (DENSITY_LIKELY(*in <= in_limit && *out <= out_limit)) {
if (DENSITY_UNLIKELY(!(state->counter & 0xf))) {
DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START;
}
state->counter++;
if (DENSITY_UNLIKELY(state->copy_penalty)) {
DENSITY_ALGORITHM_COPY(DENSITY_LION_WORK_BLOCK_SIZE);
DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START;
} else {
const uint8_t *in_start = *in;
density_lion_decode_256(in, out, &last_hash, (density_lion_dictionary *const) state->dictionary, &data, &signature, &shift);
DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*in - in_start), DENSITY_LION_WORK_BLOCK_SIZE);
}
}
if (*out > out_limit)
return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL;
read_and_decode_4:
if (DENSITY_UNLIKELY(!shift)) {
if (in_size - (*in - start) < sizeof(density_lion_signature))
return DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL;
density_lion_decode_read_signature(in, &signature);
}
form = density_lion_decode_read_form(in, &signature, &shift, &data);
switch (in_size - (*in - start)) {
case 0:
case 1:
switch (form) {
case DENSITY_LION_FORM_PLAIN:
goto process_remaining_bytes; // End marker
case DENSITY_LION_FORM_PREDICTIONS_A:
case DENSITY_LION_FORM_PREDICTIONS_B:
case DENSITY_LION_FORM_PREDICTIONS_C:
density_lion_decode_4(in, out, &last_hash, (density_lion_dictionary *const) state->dictionary, &data, form);
break;
default:
return DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING; // Not enough bytes to read a hash
}
break;
case 2:
case 3:
switch (form) {
case DENSITY_LION_FORM_PLAIN:
goto process_remaining_bytes; // End marker
default:
density_lion_decode_4(in, out, &last_hash, (density_lion_dictionary *const) state->dictionary, &data, form);
break;
}
break;
default:
density_lion_decode_4(in, out, &last_hash, (density_lion_dictionary *const) state->dictionary, &data, form);
break;
}
goto read_and_decode_4;
process_remaining_bytes:
remaining = in_size - (*in - start);
DENSITY_MEMCPY(*out, *in, remaining);
*in += remaining;
*out += remaining;
return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED;
}

54
contrib/density/src/algorithms/lion/core/lion_decode.h Normal file
View File

@ -0,0 +1,54 @@
/*
* Density
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 24/06/15 20:55
*
* --------------
* Lion algorithm
* --------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Multiform compression algorithm
*/
#ifndef DENSITY_LION_DECODE_H
#define DENSITY_LION_DECODE_H
#include "../dictionary/lion_dictionary.h"
#include "../forms/lion_form_model.h"
#include "../../algorithms.h"
DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_lion_decode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t);
#endif

298
contrib/density/src/algorithms/lion/core/lion_encode.c Normal file
View File

@ -0,0 +1,298 @@
/*
* Density
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 24/06/15 18:57
*
* --------------
* Lion algorithm
* --------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Multiform compression algorithm
*/
#include "lion_encode.h"
DENSITY_FORCE_INLINE void density_lion_encode_prepare_signature(uint8_t **DENSITY_RESTRICT out, uint_fast64_t **DENSITY_RESTRICT signature_pointer, uint_fast64_t *const DENSITY_RESTRICT signature) {
*signature = 0;
*signature_pointer = (density_lion_signature *) *out;
*out += sizeof(density_lion_signature);
}
DENSITY_FORCE_INLINE void density_lion_encode_push_to_proximity_signature(uint_fast64_t *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, const uint64_t content, const uint_fast8_t bits) {
*signature |= (content << *shift);
*shift += bits;
}
DENSITY_FORCE_INLINE void density_lion_encode_push_to_signature(uint8_t **DENSITY_RESTRICT out, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, const uint64_t content, const uint_fast8_t bits) {
if (DENSITY_LIKELY(*shift)) {
density_lion_encode_push_to_proximity_signature(signature, shift, content, bits);
if (DENSITY_UNLIKELY(*shift >= density_bitsizeof(density_lion_signature))) {
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(*signature_pointer, signature, sizeof(density_lion_signature));
#elif defined(DENSITY_BIG_ENDIAN)
const density_lion_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(*signature);
DENSITY_MEMCPY(*signature_pointer, &endian_signature, sizeof(density_lion_signature));
#else
#error
#endif
const uint_fast8_t remainder = (uint_fast8_t)(*shift & 0x3f);
*shift = 0;
if (remainder) {
density_lion_encode_prepare_signature(out, signature_pointer, signature);
density_lion_encode_push_to_proximity_signature(signature, shift, content >> (bits - remainder), remainder);
}
}
} else {
density_lion_encode_prepare_signature(out, signature_pointer, signature);
density_lion_encode_push_to_proximity_signature(signature, shift, content, bits);
}
}
DENSITY_FORCE_INLINE void density_lion_encode_push_zero_to_signature(uint8_t **DENSITY_RESTRICT out, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, const uint_fast8_t bits) {
if (DENSITY_LIKELY(*shift)) {
*shift += bits;
if (DENSITY_UNLIKELY(*shift >= density_bitsizeof(density_lion_signature))) {
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(*signature_pointer, signature, sizeof(density_lion_signature));
#elif defined(DENSITY_BIG_ENDIAN)
const density_lion_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(*signature);
DENSITY_MEMCPY(*signature_pointer, &endian_signature, sizeof(density_lion_signature));
#else
#error
#endif
const uint_fast8_t remainder = (uint_fast8_t)(*shift & 0x3f);
if (remainder) {
density_lion_encode_prepare_signature(out, signature_pointer, signature);
*shift = remainder;
} else
*shift = 0;
}
} else {
density_lion_encode_prepare_signature(out, signature_pointer, signature);
*shift = bits;
}
}
DENSITY_FORCE_INLINE void density_lion_encode_push_code_to_signature(uint8_t **DENSITY_RESTRICT out, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, const density_lion_entropy_code code) {
density_lion_encode_push_to_signature(out, signature_pointer, signature, shift, code.value, code.bitLength);
}
DENSITY_FORCE_INLINE void density_lion_encode_kernel_4(uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, const uint16_t hash, density_lion_form_data *const data, const uint32_t unit) {
density_lion_dictionary_chunk_prediction_entry *const predictions = &dictionary->predictions[*last_hash];
DENSITY_PREFETCH(&dictionary->predictions[hash]);
if (*(uint32_t *) predictions ^ unit) {
if (*((uint32_t *) predictions + 1) ^ unit) {
if (*((uint32_t *) predictions + 2) ^ unit) {
density_lion_dictionary_chunk_entry *const in_dictionary = &dictionary->chunks[hash];
if (*(uint32_t *) in_dictionary ^ unit) {
if (*((uint32_t *) in_dictionary + 1) ^ unit) {
if (*((uint32_t *) in_dictionary + 2) ^ unit) {
if (*((uint32_t *) in_dictionary + 3) ^ unit) {
density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_PLAIN));
DENSITY_MEMCPY(*out, &unit, sizeof(uint32_t));
*out += sizeof(uint32_t);
} else {
density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_DICTIONARY_D));
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t));
#elif defined(DENSITY_BIG_ENDIAN)
const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash);
DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t));
#else
#error
#endif
*out += sizeof(uint16_t);
}
} else {
density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_DICTIONARY_C));
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t));
#elif defined(DENSITY_BIG_ENDIAN)
const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash);
DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t));
#else
#error
#endif
*out += sizeof(uint16_t);
}
} else {
density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_DICTIONARY_B));
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t));
#elif defined(DENSITY_BIG_ENDIAN)
const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash);
DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t));
#else
#error
#endif
*out += sizeof(uint16_t);
}
DENSITY_MEMMOVE((uint32_t *) in_dictionary + 1, in_dictionary, 3 * sizeof(uint32_t));
*(uint32_t *) in_dictionary = unit; // Does not ensure dictionary content consistency between endiannesses
} else {
density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_DICTIONARY_A));
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(*out, &hash, sizeof(uint16_t));
#elif defined(DENSITY_BIG_ENDIAN)
const uint16_t endian_hash = DENSITY_LITTLE_ENDIAN_16(hash);
DENSITY_MEMCPY(*out, &endian_hash, sizeof(uint16_t));
#else
#error
#endif
*out += sizeof(uint16_t);
}
} else {
density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_PREDICTIONS_C));
}
} else {
density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_PREDICTIONS_B));
}
DENSITY_MEMMOVE((uint32_t *) predictions + 1, predictions, 2 * sizeof(uint32_t));
*(uint32_t *) predictions = unit; // Does not ensure dictionary content consistency between endiannesses
} else
density_lion_encode_push_code_to_signature(out, signature_pointer, signature, shift, density_lion_form_model_get_encoding(data, DENSITY_LION_FORM_PREDICTIONS_A));
*last_hash = hash;
}
DENSITY_FORCE_INLINE void density_lion_encode_4(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const data, uint32_t *DENSITY_RESTRICT unit) {
DENSITY_MEMCPY(unit, *in, sizeof(uint32_t));
density_lion_encode_kernel_4(out, last_hash, signature_pointer, signature, shift, dictionary, DENSITY_LION_HASH_ALGORITHM(DENSITY_LITTLE_ENDIAN_32(*unit)), data, *unit);
*in += sizeof(uint32_t);
}
DENSITY_FORCE_INLINE void density_lion_encode_generic(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, const uint_fast8_t chunks_per_process_unit, density_lion_form_data *const data, uint32_t *DENSITY_RESTRICT unit) {
#ifdef __clang__
for (uint_fast8_t count = 0; count < (chunks_per_process_unit >> 2); count++) {
DENSITY_UNROLL_4(density_lion_encode_4(in, out, last_hash, signature_pointer, signature, shift, dictionary, data, unit));
}
#else
for (uint_fast8_t count = 0; count < (chunks_per_process_unit >> 1); count++) {
DENSITY_UNROLL_2(density_lion_encode_4(in, out, last_hash, signature_pointer, signature, shift, dictionary, data, unit));
}
#endif
}
DENSITY_FORCE_INLINE void density_lion_encode_32(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const data, uint32_t *DENSITY_RESTRICT unit) {
density_lion_encode_generic(in, out, last_hash, signature_pointer, signature, shift, dictionary, DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_SMALL, data, unit);
}
DENSITY_FORCE_INLINE void density_lion_encode_256(const uint8_t **DENSITY_RESTRICT in, uint8_t **DENSITY_RESTRICT out, uint_fast16_t *DENSITY_RESTRICT last_hash, density_lion_signature **DENSITY_RESTRICT signature_pointer, density_lion_signature *const DENSITY_RESTRICT signature, uint_fast8_t *const DENSITY_RESTRICT shift, density_lion_dictionary *const DENSITY_RESTRICT dictionary, density_lion_form_data *const data, uint32_t *DENSITY_RESTRICT unit) {
density_lion_encode_generic(in, out, last_hash, signature_pointer, signature, shift, dictionary, DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG, data, unit);
}
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_algorithm_exit_status density_lion_encode(density_algorithm_state *const DENSITY_RESTRICT state, const uint8_t **DENSITY_RESTRICT in, const uint_fast64_t in_size, uint8_t **DENSITY_RESTRICT out, const uint_fast64_t out_size) {
if (out_size < DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE)
return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL;
density_lion_signature signature = 0;
density_lion_signature *signature_pointer = NULL;
uint_fast8_t shift = 0;
density_lion_form_data data;
density_lion_form_model_init(&data);
uint_fast16_t last_hash = 0;
uint32_t unit;
uint8_t *out_limit = *out + out_size - DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE;
uint_fast64_t limit_256 = (in_size >> 8);
while (DENSITY_LIKELY(limit_256-- && *out <= out_limit)) {
if (DENSITY_UNLIKELY(!(state->counter & 0xf))) {
DENSITY_ALGORITHM_REDUCE_COPY_PENALTY_START;
}
state->counter++;
if (DENSITY_UNLIKELY(state->copy_penalty)) {
DENSITY_ALGORITHM_COPY(DENSITY_LION_WORK_BLOCK_SIZE);
DENSITY_ALGORITHM_INCREASE_COPY_PENALTY_START;
} else {
const uint8_t *out_start = *out;
DENSITY_PREFETCH(*in + DENSITY_LION_WORK_BLOCK_SIZE);
density_lion_encode_256(in, out, &last_hash, &signature_pointer, &signature, &shift, (density_lion_dictionary *const) state->dictionary, &data, &unit);
DENSITY_ALGORITHM_TEST_INCOMPRESSIBILITY((*out - out_start), DENSITY_LION_WORK_BLOCK_SIZE);
}
}
if (*out > out_limit)
return DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL;
uint_fast64_t remaining;
switch (in_size & 0xff) {
case 0:
case 1:
case 2:
case 3:
density_lion_encode_push_code_to_signature(out, &signature_pointer, &signature, &shift, density_lion_form_model_get_encoding(&data, DENSITY_LION_FORM_PLAIN)); // End marker
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_lion_signature));
#elif defined(DENSITY_BIG_ENDIAN)
const density_lion_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature);
DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_lion_signature));
#else
#error
#endif
goto process_remaining_bytes;
default:
break;
}
uint_fast64_t limit_4 = (in_size & 0xff) >> 2;
while (limit_4--)
density_lion_encode_4(in, out, &last_hash, &signature_pointer, &signature, &shift, (density_lion_dictionary *const) state->dictionary, &data, &unit);
density_lion_encode_push_code_to_signature(out, &signature_pointer, &signature, &shift, density_lion_form_model_get_encoding(&data, DENSITY_LION_FORM_PLAIN)); // End marker
#ifdef DENSITY_LITTLE_ENDIAN
DENSITY_MEMCPY(signature_pointer, &signature, sizeof(density_lion_signature));
#elif defined(DENSITY_BIG_ENDIAN)
const density_lion_signature endian_signature = DENSITY_LITTLE_ENDIAN_64(signature);
DENSITY_MEMCPY(signature_pointer, &endian_signature, sizeof(density_lion_signature));
#else
#error
#endif
process_remaining_bytes:
remaining = in_size & 0x3;
if (remaining) {
DENSITY_MEMCPY(*out, *in, remaining);
*in += remaining;
*out += remaining;
}
return DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED;
}

54
contrib/density/src/algorithms/lion/core/lion_encode.h Normal file
View File

@ -0,0 +1,54 @@
/*
* Density
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 24/06/15 18:56
*
* --------------
* Lion algorithm
* --------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Multiform compression algorithm
*/
#ifndef DENSITY_LION_ENCODE_H
#define DENSITY_LION_ENCODE_H
#include "../dictionary/lion_dictionary.h"
#include "../forms/lion_form_model.h"
#include "../../algorithms.h"
DENSITY_WINDOWS_EXPORT density_algorithm_exit_status density_lion_encode(density_algorithm_state *const DENSITY_RESTRICT_DECLARE, const uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t, uint8_t **DENSITY_RESTRICT_DECLARE, const uint_fast64_t);
#endif

73
contrib/density/src/algorithms/lion/dictionary/lion_dictionary.h Normal file
View File

@ -0,0 +1,73 @@
/*
* Density
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 12/02/15 23:09
*
* --------------
* Lion algorithm
* --------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Multiform compression algorithm
*/
#ifndef DENSITY_LION_DICTIONARY_H
#define DENSITY_LION_DICTIONARY_H
#include "../lion.h"
#pragma pack(push)
#pragma pack(4)
typedef struct {
uint32_t chunk_a;
uint32_t chunk_b;
uint32_t chunk_c;
uint32_t chunk_d;
uint32_t chunk_e;
} density_lion_dictionary_chunk_entry;
typedef struct {
uint32_t next_chunk_a;
uint32_t next_chunk_b;
uint32_t next_chunk_c;
} density_lion_dictionary_chunk_prediction_entry;
typedef struct {
density_lion_dictionary_chunk_entry chunks[1 << DENSITY_LION_CHUNK_HASH_BITS];
density_lion_dictionary_chunk_prediction_entry predictions[1 << DENSITY_LION_CHUNK_HASH_BITS];
} density_lion_dictionary;
#pragma pack(pop)
#endif

153
contrib/density/src/algorithms/lion/forms/lion_form_model.c Normal file
View File

@ -0,0 +1,153 @@
/*
* Density
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 9/03/15 11:19
*
* --------------
* Lion algorithm
* --------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Multiform compression algorithm
*/
#include "lion_form_model.h"
const density_lion_entropy_code density_lion_form_entropy_codes[DENSITY_LION_NUMBER_OF_FORMS] = DENSITY_LION_FORM_MODEL_ENTROPY_CODES;
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_lion_form_model_init(density_lion_form_data *const data) {
density_lion_form_node *rank_0 = &data->formsPool[0];
rank_0->form = DENSITY_LION_FORM_PLAIN;
rank_0->rank = 0;
rank_0->previousForm = NULL;
data->formsIndex[DENSITY_LION_FORM_PLAIN] = rank_0;
density_lion_form_node *rank_1 = &data->formsPool[1];
rank_1->form = DENSITY_LION_FORM_DICTIONARY_A;
rank_1->rank = 1;
rank_1->previousForm = rank_0;
data->formsIndex[DENSITY_LION_FORM_DICTIONARY_A] = rank_1;
density_lion_form_node *rank_2 = &data->formsPool[2];
rank_2->form = DENSITY_LION_FORM_DICTIONARY_B;
rank_2->rank = 2;
rank_2->previousForm = rank_1;
data->formsIndex[DENSITY_LION_FORM_DICTIONARY_B] = rank_2;
density_lion_form_node *rank_3 = &data->formsPool[3];
rank_3->form = DENSITY_LION_FORM_PREDICTIONS_A;
rank_3->rank = 3;
rank_3->previousForm = rank_2;
data->formsIndex[DENSITY_LION_FORM_PREDICTIONS_A] = rank_3;
density_lion_form_node *rank_4 = &data->formsPool[4];
rank_4->form = DENSITY_LION_FORM_PREDICTIONS_B;
rank_4->rank = 4;
rank_4->previousForm = rank_3;
data->formsIndex[DENSITY_LION_FORM_PREDICTIONS_B] = rank_4;
density_lion_form_node *rank_5 = &data->formsPool[5];
rank_5->form = DENSITY_LION_FORM_DICTIONARY_C;
rank_5->rank = 5;
rank_5->previousForm = rank_4;
data->formsIndex[DENSITY_LION_FORM_DICTIONARY_C] = rank_5;
density_lion_form_node *rank_6 = &data->formsPool[6];
rank_6->form = DENSITY_LION_FORM_PREDICTIONS_C;
rank_6->rank = 6;
rank_6->previousForm = rank_5;
data->formsIndex[DENSITY_LION_FORM_PREDICTIONS_C] = rank_6;
density_lion_form_node *rank_7 = &data->formsPool[7];
rank_7->form = DENSITY_LION_FORM_DICTIONARY_D;
rank_7->rank = 7;
rank_7->previousForm = rank_6;
data->formsIndex[DENSITY_LION_FORM_DICTIONARY_D] = rank_7;
data->usages.usages_as_uint64_t = 0;
}
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_lion_form_model_attach(density_lion_form_data *const data, void (*attachments[DENSITY_LION_NUMBER_OF_FORMS])(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const)) {
for(uint_fast8_t count = 0; count < DENSITY_LION_NUMBER_OF_FORMS; count ++)
data->attachments[count] = attachments[count];
}
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_lion_form_model_update(density_lion_form_data *const DENSITY_RESTRICT data, density_lion_form_node *const DENSITY_RESTRICT form, const uint8_t usage, density_lion_form_node *const DENSITY_RESTRICT previous_form, const uint8_t previous_usage) {
if (DENSITY_UNLIKELY(previous_usage < usage)) { // Relative stability is assumed
const DENSITY_LION_FORM form_value = form->form;
const DENSITY_LION_FORM previous_form_value = previous_form->form;
previous_form->form = form_value;
form->form = previous_form_value;
data->formsIndex[form_value] = previous_form;
data->formsIndex[previous_form_value] = form;
}
}
DENSITY_FORCE_INLINE void density_lion_form_model_flatten(density_lion_form_data *const data, const uint8_t usage) {
if (DENSITY_UNLIKELY(usage & 0x80))
data->usages.usages_as_uint64_t = (data->usages.usages_as_uint64_t >> 1) & 0x7f7f7f7f7f7f7f7fllu; // Flatten usage values
}
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE DENSITY_LION_FORM density_lion_form_model_increment_usage(density_lion_form_data *const data, density_lion_form_node *const DENSITY_RESTRICT form) {
const DENSITY_LION_FORM form_value = form->form;
const uint8_t usage = ++data->usages.usages_as_uint8_t[form_value];
density_lion_form_node *const previous_form = form->previousForm;
if (previous_form)
density_lion_form_model_update(data, form, usage, previous_form, data->usages.usages_as_uint8_t[previous_form->form]);
else
density_lion_form_model_flatten(data, usage);
return form_value;
}
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE density_lion_entropy_code density_lion_form_model_get_encoding(density_lion_form_data *const data, const DENSITY_LION_FORM form) {
const uint8_t usage = ++data->usages.usages_as_uint8_t[form];
density_lion_form_node *const form_found = data->formsIndex[form];
density_lion_form_node *const previous_form = form_found->previousForm;
if (previous_form) {
density_lion_form_model_update(data, form_found, usage, previous_form, data->usages.usages_as_uint8_t[previous_form->form]);
return density_lion_form_entropy_codes[form_found->rank];
} else {
density_lion_form_model_flatten(data, usage);
return density_lion_form_entropy_codes[0];
}
}

96
contrib/density/src/algorithms/lion/forms/lion_form_model.h Normal file
View File

@ -0,0 +1,96 @@
/*
* Density
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 9/03/15 12:04
*
* --------------
* Lion algorithm
* --------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Multiform compression algorithm
*/
#ifndef DENSITY_LION_FORM_MODEL_H
#define DENSITY_LION_FORM_MODEL_H
#include "../../../globals.h"
#include "../lion.h"
#define DENSITY_LION_NUMBER_OF_FORMS 8
// Unary codes (reversed) except the last one
#define DENSITY_LION_FORM_MODEL_ENTROPY_CODES {\
{DENSITY_BINARY_TO_UINT(1), 1},\
{DENSITY_BINARY_TO_UINT(10), 2},\
{DENSITY_BINARY_TO_UINT(100), 3},\
{DENSITY_BINARY_TO_UINT(1000), 4},\
{DENSITY_BINARY_TO_UINT(10000), 5},\
{DENSITY_BINARY_TO_UINT(100000), 6},\
{DENSITY_BINARY_TO_UINT(1000000), 7},\
{DENSITY_BINARY_TO_UINT(0000000), 7},\
}
#pragma pack(push)
#pragma pack(4)
typedef struct {
void* previousForm;
DENSITY_LION_FORM form;
uint8_t rank;
} density_lion_form_node;
typedef struct {
union {
uint8_t usages_as_uint8_t[DENSITY_LION_NUMBER_OF_FORMS];
uint64_t usages_as_uint64_t;
} usages;
void (*attachments[DENSITY_LION_NUMBER_OF_FORMS])(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const);
density_lion_form_node formsPool[DENSITY_LION_NUMBER_OF_FORMS];
density_lion_form_node *formsIndex[DENSITY_LION_NUMBER_OF_FORMS];
uint8_t nextAvailableForm;
} density_lion_form_data;
#pragma pack(pop)
DENSITY_WINDOWS_EXPORT void density_lion_form_model_init(density_lion_form_data *const);
DENSITY_WINDOWS_EXPORT void density_lion_form_model_attach(density_lion_form_data *const, void (*[DENSITY_LION_NUMBER_OF_FORMS])(const uint8_t **, uint8_t **, uint_fast16_t *, void *const, uint16_t *const, uint32_t *const));
DENSITY_WINDOWS_EXPORT void density_lion_form_model_update(density_lion_form_data *const DENSITY_RESTRICT_DECLARE, density_lion_form_node *const DENSITY_RESTRICT_DECLARE, const uint8_t, density_lion_form_node *const DENSITY_RESTRICT_DECLARE, const uint8_t);
DENSITY_WINDOWS_EXPORT DENSITY_LION_FORM density_lion_form_model_increment_usage(density_lion_form_data *const, density_lion_form_node *const DENSITY_RESTRICT_DECLARE);
DENSITY_WINDOWS_EXPORT density_lion_entropy_code density_lion_form_model_get_encoding(density_lion_form_data *const, const DENSITY_LION_FORM);
#endif

94
contrib/density/src/algorithms/lion/lion.h Normal file
View File

@ -0,0 +1,94 @@
/*
* Density
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 5/02/15 20:57
*
* --------------
* Lion algorithm
* --------------
*
* Author(s)
* Guillaume Voirin (https://github.com/gpnuma)
*
* Description
* Multiform compression algorithm
*/
#ifndef DENSITY_LION_H
#define DENSITY_LION_H
#include "../../globals.h"
#define DENSITY_LION_HASH32_MULTIPLIER (uint32_t)0x9D6EF916lu
#define DENSITY_LION_CHUNK_HASH_BITS 16
#define DENSITY_LION_HASH_ALGORITHM(value32) (uint16_t)(value32 * DENSITY_LION_HASH32_MULTIPLIER >> (32 - DENSITY_LION_CHUNK_HASH_BITS))
typedef enum {
DENSITY_LION_FORM_PREDICTIONS_A = 0,
DENSITY_LION_FORM_PREDICTIONS_B,
DENSITY_LION_FORM_PREDICTIONS_C,
DENSITY_LION_FORM_DICTIONARY_A,
DENSITY_LION_FORM_DICTIONARY_B,
DENSITY_LION_FORM_DICTIONARY_C,
DENSITY_LION_FORM_DICTIONARY_D,
DENSITY_LION_FORM_PLAIN,
} DENSITY_LION_FORM;
typedef enum {
DENSITY_LION_PREDICTIONS_SIGNATURE_FLAG_A = 0x0,
DENSITY_LION_PREDICTIONS_SIGNATURE_FLAG_B = 0x1,
} DENSITY_LION_PREDICTIONS_SIGNATURE_FLAG;
#pragma pack(push)
#pragma pack(4)
typedef struct {
uint_fast8_t value;
uint_fast8_t bitLength;
} density_lion_entropy_code;
#pragma pack(pop)
typedef uint64_t density_lion_signature;
#define DENSITY_LION_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE (density_bitsizeof(density_lion_signature) * sizeof(uint32_t)) // Plain writes
#define DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE (sizeof(density_lion_signature) + DENSITY_LION_MAXIMUM_COMPRESSED_BODY_SIZE_PER_SIGNATURE)
#define DENSITY_LION_MAXIMUM_DECOMPRESSED_UNIT_SIZE (density_bitsizeof(density_lion_signature) * sizeof(uint32_t)) // Smallest form size times work unit size
#define DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_SMALL 8
#define DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG 64
#define DENSITY_LION_PROCESS_UNIT_SIZE_SMALL (DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_SMALL * sizeof(uint32_t))
#define DENSITY_LION_PROCESS_UNIT_SIZE_BIG (DENSITY_LION_CHUNKS_PER_PROCESS_UNIT_BIG * sizeof(uint32_t))
#define DENSITY_LION_WORK_BLOCK_SIZE 256
#define DENSITY_LION_COPY_PENALTY 2
#endif

220
contrib/density/src/buffers/buffer.c Normal file
View File

@ -0,0 +1,220 @@
/*
* Density
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 3/02/15 19:53
*/
#include "buffer.h"
DENSITY_WINDOWS_EXPORT uint_fast64_t density_compress_safe_size(const uint_fast64_t input_size) {
const uint_fast64_t slack = DENSITY_MAX_3(DENSITY_CHAMELEON_MAXIMUM_COMPRESSED_UNIT_SIZE, DENSITY_CHEETAH_MAXIMUM_COMPRESSED_UNIT_SIZE, DENSITY_LION_MAXIMUM_COMPRESSED_UNIT_SIZE);
// Chameleon longest output
uint_fast64_t chameleon_longest_output_size = 0;
chameleon_longest_output_size += sizeof(density_header);
chameleon_longest_output_size += sizeof(density_chameleon_signature) * (1 + (input_size >> (5 + 3))); // Signature space (1 bit <=> 4 bytes)
chameleon_longest_output_size += sizeof(density_chameleon_signature); // Eventual supplementary signature for end marker
chameleon_longest_output_size += input_size; // Everything encoded as plain data
// Cheetah longest output
uint_fast64_t cheetah_longest_output_size = 0;
cheetah_longest_output_size += sizeof(density_header);
cheetah_longest_output_size += sizeof(density_cheetah_signature) * (1 + (input_size >> (4 + 3))); // Signature space (2 bits <=> 4 bytes)
cheetah_longest_output_size += sizeof(density_cheetah_signature); // Eventual supplementary signature for end marker
cheetah_longest_output_size += input_size; // Everything encoded as plain data
// Lion longest output
uint_fast64_t lion_longest_output_size = 0;
lion_longest_output_size += sizeof(density_header);
lion_longest_output_size += sizeof(density_lion_signature) * (1 + ((input_size * 7) >> (5 + 3))); // Signature space (7 bits <=> 4 bytes), although this size is technically impossible
lion_longest_output_size += sizeof(density_lion_signature); // Eventual supplementary signature for end marker
lion_longest_output_size += input_size; // Everything encoded as plain data
return DENSITY_MAX_3(chameleon_longest_output_size, cheetah_longest_output_size, lion_longest_output_size) + slack;
}
DENSITY_WINDOWS_EXPORT uint_fast64_t density_decompress_safe_size(const uint_fast64_t expected_decompressed_output_size) {
const uint_fast64_t slack = DENSITY_MAX_3(DENSITY_CHAMELEON_DECOMPRESSED_UNIT_SIZE, DENSITY_CHEETAH_DECOMPRESSED_UNIT_SIZE, DENSITY_LION_MAXIMUM_DECOMPRESSED_UNIT_SIZE);
return expected_decompressed_output_size + slack;
}
DENSITY_FORCE_INLINE DENSITY_STATE density_convert_algorithm_exit_status(const density_algorithm_exit_status status) {
switch (status) {
case DENSITY_ALGORITHMS_EXIT_STATUS_FINISHED:
return DENSITY_STATE_OK;
case DENSITY_ALGORITHMS_EXIT_STATUS_INPUT_STALL:
return DENSITY_STATE_ERROR_INPUT_BUFFER_TOO_SMALL;
case DENSITY_ALGORITHMS_EXIT_STATUS_OUTPUT_STALL:
return DENSITY_STATE_ERROR_OUTPUT_BUFFER_TOO_SMALL;
default:
return DENSITY_STATE_ERROR_DURING_PROCESSING;
}
}
DENSITY_FORCE_INLINE density_processing_result density_make_result(const DENSITY_STATE state, const uint_fast64_t read, const uint_fast64_t written, density_context *const context) {
density_processing_result result;
result.state = state;
result.bytesRead = read;
result.bytesWritten = written;
result.context = context;
return result;
}
DENSITY_FORCE_INLINE density_context* density_allocate_context(const DENSITY_ALGORITHM algorithm, const bool custom_dictionary, void *(*mem_alloc)(size_t)) {
density_context* context = mem_alloc(sizeof(density_context));
context->algorithm = algorithm;
context->dictionary_size = density_get_dictionary_size(context->algorithm);
context->dictionary_type = custom_dictionary;
if(!context->dictionary_type) {
context->dictionary = mem_alloc(context->dictionary_size);
DENSITY_MEMSET(context->dictionary, 0, context->dictionary_size);
}
return context;
}
DENSITY_WINDOWS_EXPORT void density_free_context(density_context *const context, void (*mem_free)(void *)) {
if(mem_free == NULL)
mem_free = free;
if(!context->dictionary_type)
mem_free(context->dictionary);
mem_free(context);
}
DENSITY_WINDOWS_EXPORT density_processing_result density_compress_prepare_context(const DENSITY_ALGORITHM algorithm, const bool custom_dictionary, void *(*mem_alloc)(size_t)) {
if(mem_alloc == NULL)
mem_alloc = malloc;
return density_make_result(DENSITY_STATE_OK, 0, 0, density_allocate_context(algorithm, custom_dictionary, mem_alloc));
}
DENSITY_WINDOWS_EXPORT density_processing_result density_compress_with_context(const uint8_t * input_buffer, const uint_fast64_t input_size, uint8_t * output_buffer, const uint_fast64_t output_size, density_context *const context) {
if (output_size < sizeof(density_header))
return density_make_result(DENSITY_STATE_ERROR_OUTPUT_BUFFER_TOO_SMALL, 0, 0, context);
if(context == NULL)
return density_make_result(DENSITY_STATE_ERROR_INVALID_CONTEXT, 0, 0, context);
// Variables setup
const uint8_t *in = input_buffer;
uint8_t *out = output_buffer;
density_algorithm_state state;
density_algorithm_exit_status status = DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING;
// Header
density_header_write(&out, context->algorithm);
// Compression
density_algorithms_prepare_state(&state, context->dictionary);
switch (context->algorithm) {
case DENSITY_ALGORITHM_CHAMELEON:
status = density_chameleon_encode(&state, &in, input_size, &out, output_size);
break;
case DENSITY_ALGORITHM_CHEETAH:
status = density_cheetah_encode(&state, &in, input_size, &out, output_size);
break;
case DENSITY_ALGORITHM_LION:
status = density_lion_encode(&state, &in, input_size, &out, output_size);
break;
}
// Result
return density_make_result(density_convert_algorithm_exit_status(status), in - input_buffer, out - output_buffer, context);
}
DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_prepare_context(const uint8_t *input_buffer, const uint_fast64_t input_size, const bool custom_dictionary, void *(*mem_alloc)(size_t)) {
if (input_size < sizeof(density_header))
return density_make_result(DENSITY_STATE_ERROR_INPUT_BUFFER_TOO_SMALL, 0, 0, NULL);
// Variables setup
const uint8_t* in = input_buffer;
if(mem_alloc == NULL)
mem_alloc = malloc;
// Read header
density_header main_header;
density_header_read(&in, &main_header);
// Setup context
density_context *const context = density_allocate_context(main_header.algorithm, custom_dictionary, mem_alloc);
return density_make_result(DENSITY_STATE_OK, in - input_buffer, 0, context);
}
DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_with_context(const uint8_t * input_buffer, const uint_fast64_t input_size, uint8_t * output_buffer, const uint_fast64_t output_size, density_context *const context) {
if(context == NULL)
return density_make_result(DENSITY_STATE_ERROR_INVALID_CONTEXT, 0, 0, context);
// Variables setup
const uint8_t *in = input_buffer;
uint8_t *out = output_buffer;
density_algorithm_state state;
density_algorithm_exit_status status = DENSITY_ALGORITHMS_EXIT_STATUS_ERROR_DURING_PROCESSING;
// Decompression
density_algorithms_prepare_state(&state, context->dictionary);
switch (context->algorithm) {
case DENSITY_ALGORITHM_CHAMELEON:
status = density_chameleon_decode(&state, &in, input_size, &out, output_size);
break;
case DENSITY_ALGORITHM_CHEETAH:
status = density_cheetah_decode(&state, &in, input_size, &out, output_size);
break;
case DENSITY_ALGORITHM_LION:
status = density_lion_decode(&state, &in, input_size, &out, output_size);
break;
}
// Result
return density_make_result(density_convert_algorithm_exit_status(status), in - input_buffer, out - output_buffer, context);
}
DENSITY_WINDOWS_EXPORT density_processing_result density_compress(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size, const DENSITY_ALGORITHM algorithm) {
density_processing_result result = density_compress_prepare_context(algorithm, false, malloc);
if(result.state) {
density_free_context(result.context, free);
return result;
}
result = density_compress_with_context(input_buffer, input_size, output_buffer, output_size, result.context);
density_free_context(result.context, free);
return result;
}
DENSITY_WINDOWS_EXPORT density_processing_result density_decompress(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size) {
density_processing_result result = density_decompress_prepare_context(input_buffer, input_size, false, malloc);
if(result.state) {
density_free_context(result.context, free);
return result;
}
result = density_decompress_with_context(input_buffer + result.bytesRead, input_size - result.bytesRead, output_buffer, output_size, result.context);
density_free_context(result.context, free);
return result;
}

58
contrib/density/src/buffers/buffer.h Normal file
View File

@ -0,0 +1,58 @@
/*
* Density
*
* Copyright (c) 2015, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 3/02/15 19:51
*/
#ifndef DENSITY_BUFFER_H
#define DENSITY_BUFFER_H
#include "../globals.h"
#include "../density_api.h"
#include "../structure/header.h"
#include "../algorithms/chameleon/core/chameleon_encode.h"
#include "../algorithms/chameleon/core/chameleon_decode.h"
#include "../algorithms/cheetah/core/cheetah_encode.h"
#include "../algorithms/cheetah/core/cheetah_decode.h"
#include "../algorithms/lion/core/lion_encode.h"
#include "../algorithms/lion/core/lion_decode.h"
DENSITY_WINDOWS_EXPORT uint_fast64_t density_compress_safe_size(const uint_fast64_t);
DENSITY_WINDOWS_EXPORT uint_fast64_t density_decompress_safe_size(const uint_fast64_t);
DENSITY_WINDOWS_EXPORT void density_free_context(density_context *const, void (*)(void *));
DENSITY_WINDOWS_EXPORT density_processing_result density_compress_prepare_context(const DENSITY_ALGORITHM, const bool, void *(*)(size_t));
DENSITY_WINDOWS_EXPORT density_processing_result density_compress_with_context(const uint8_t *, const uint_fast64_t, uint8_t *, const uint_fast64_t, density_context *const);
DENSITY_WINDOWS_EXPORT density_processing_result density_compress(const uint8_t *, const uint_fast64_t, uint8_t *, const uint_fast64_t, const DENSITY_ALGORITHM);
DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_prepare_context(const uint8_t *, const uint_fast64_t, const bool, void *(*)(size_t));
DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_with_context(const uint8_t *, const uint_fast64_t, uint8_t *, const uint_fast64_t, density_context *const);
DENSITY_WINDOWS_EXPORT density_processing_result density_decompress(const uint8_t *, const uint_fast64_t, uint8_t *, const uint_fast64_t);
#endif

220
contrib/density/src/density_api.h Normal file
View File

@ -0,0 +1,220 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 18/10/13 22:41
*/
#ifndef DENSITY_API_H
#define DENSITY_API_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include <stdint.h>
#include <stdbool.h>
#if defined(_WIN64) || defined(_WIN32)
#define DENSITY_WINDOWS_EXPORT __declspec(dllexport)
#else
#define DENSITY_WINDOWS_EXPORT
#endif
/***********************************************************************************************************************
* *
* API data structures *
* *
***********************************************************************************************************************/
typedef uint8_t density_byte;
typedef bool density_bool;
typedef enum {
DENSITY_ALGORITHM_CHAMELEON = 1,
DENSITY_ALGORITHM_CHEETAH = 2,
DENSITY_ALGORITHM_LION = 3,
} DENSITY_ALGORITHM;
typedef enum {
DENSITY_STATE_OK = 0, // Everything went alright
DENSITY_STATE_ERROR_INPUT_BUFFER_TOO_SMALL, // Input buffer size is too small
DENSITY_STATE_ERROR_OUTPUT_BUFFER_TOO_SMALL, // Output buffer size is too small
DENSITY_STATE_ERROR_DURING_PROCESSING, // Error during processing
DENSITY_STATE_ERROR_INVALID_CONTEXT, // Invalid context
DENSITY_STATE_ERROR_INVALID_ALGORITHM, // Invalid algorithm
} DENSITY_STATE;
typedef struct {
DENSITY_ALGORITHM algorithm;
bool dictionary_type;
size_t dictionary_size;
void* dictionary;
} density_context;
typedef struct {
DENSITY_STATE state;
uint_fast64_t bytesRead;
uint_fast64_t bytesWritten;
density_context* context;
} density_processing_result;
/***********************************************************************************************************************
* *
* Density version information *
* *
***********************************************************************************************************************/
/*
* Returns the major density version
*/
DENSITY_WINDOWS_EXPORT uint8_t density_version_major(void);
/*
* Returns the minor density version
*/
DENSITY_WINDOWS_EXPORT uint8_t density_version_minor(void);
/*
* Returns the density revision
*/
DENSITY_WINDOWS_EXPORT uint8_t density_version_revision(void);
/***********************************************************************************************************************
* *
* Density API functions *
* *
***********************************************************************************************************************/
/*
* Return the required size of an algorithm's dictionary
*
* @param algorithm the algorithm to use this dictionary for
*/
DENSITY_WINDOWS_EXPORT size_t density_get_dictionary_size(DENSITY_ALGORITHM algorithm);
/*
* Return an output buffer byte size which guarantees enough space for encoding input_size bytes
*
* @param input_size the size of the input data which is about to be compressed
*/
DENSITY_WINDOWS_EXPORT uint_fast64_t density_compress_safe_size(const uint_fast64_t input_size);
/*
* Return an output buffer byte size which, if expected_decompressed_output_size is correct, will enable density to decompress properly
*
* @param expected_decompressed_output_size the expected (original) size of the decompressed data
*/
DENSITY_WINDOWS_EXPORT uint_fast64_t density_decompress_safe_size(const uint_fast64_t expected_decompressed_output_size);
/*
* Releases a context from memory.
*
* @param context the context to free
* @param mem_free the memory freeing function. If set to NULL, free() is used
*/
DENSITY_WINDOWS_EXPORT void density_free_context(density_context *const context, void (*mem_free)(void *));
/*
* Allocate a context in memory using the provided function and optional dictionary
*
* @param algorithm the required algorithm
* @param custom_dictionary use an eventual custom dictionary ? If set to true the context's dictionary will have to be allocated
* @param mem_alloc the memory allocation function. If set to NULL, malloc() is used
*/
DENSITY_WINDOWS_EXPORT density_processing_result density_compress_prepare_context(const DENSITY_ALGORITHM algorithm, const bool custom_dictionary, void *(*mem_alloc)(size_t));
/*
* Compress an input_buffer of input_size bytes and store the result in output_buffer, using the provided context.
* Important note * this function could be unsafe memory-wise if not used properly.
*
* @param input_buffer a buffer of bytes
* @param input_size the size in bytes of input_buffer
* @param output_buffer a buffer of bytes
* @param output_size the size of output_buffer, must be at least DENSITY_MINIMUM_OUTPUT_BUFFER_SIZE
* @param context a pointer to a context structure
*/
DENSITY_WINDOWS_EXPORT density_processing_result density_compress_with_context(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size, density_context *const context);
/*
* Compress an input_buffer of input_size bytes and store the result in output_buffer.
*
* @param input_buffer a buffer of bytes
* @param input_size the size in bytes of input_buffer
* @param output_buffer a buffer of bytes
* @param output_size the size of output_buffer, must be at least DENSITY_MINIMUM_OUTPUT_BUFFER_SIZE
* @param algorithm the algorithm to use
*/
DENSITY_WINDOWS_EXPORT density_processing_result density_compress(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size, const DENSITY_ALGORITHM algorithm);
/*
* Reads the compressed data's header and creates an adequate decompression context.
*
* @param input_buffer a buffer of bytes
* @param input_size the size in bytes of input_buffer
* @param custom_dictionary use a custom dictionary ? If set to true the context's dictionary will have to be allocated
* @param mem_alloc the memory allocation function. If set to NULL, malloc() is used
*/
DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_prepare_context(const uint8_t *input_buffer, const uint_fast64_t input_size, const bool custom_dictionary, void *(*mem_alloc)(size_t));
/*
* Decompress an input_buffer of input_size bytes and store the result in output_buffer, using the provided dictionary.
* Important notes * You must know in advance the algorithm used for compression to provide the proper dictionary.
* * This function could be unsafe memory-wise if not used properly.
*
* @param input_buffer a buffer of bytes
* @param input_size the size in bytes of input_buffer
* @param output_buffer a buffer of bytes
* @param output_size the size of output_buffer, must be at least DENSITY_MINIMUM_OUTPUT_BUFFER_SIZE
* @param dictionaries a pointer to a dictionary
*/
DENSITY_WINDOWS_EXPORT density_processing_result density_decompress_with_context(const uint8_t * input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size, density_context *const context);
/*
* Decompress an input_buffer of input_size bytes and store the result in output_buffer.
*
* @param input_buffer a buffer of bytes
* @param input_size the size in bytes of input_buffer
* @param output_buffer a buffer of bytes
* @param output_size the size of output_buffer, must be at least DENSITY_MINIMUM_OUTPUT_BUFFER_SIZE
*/
DENSITY_WINDOWS_EXPORT density_processing_result density_decompress(const uint8_t *input_buffer, const uint_fast64_t input_size, uint8_t *output_buffer, const uint_fast64_t output_size);
#ifdef __cplusplus
}
#endif
#endif

47
contrib/density/src/globals.c Normal file
View File

@ -0,0 +1,47 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 01/11/13 13:39
*/
#include "globals.h"
DENSITY_WINDOWS_EXPORT uint8_t density_version_major() {
return DENSITY_MAJOR_VERSION;
}
DENSITY_WINDOWS_EXPORT uint8_t density_version_minor() {
return DENSITY_MINOR_VERSION;
}
DENSITY_WINDOWS_EXPORT uint8_t density_version_revision() {
return DENSITY_REVISION;
}

232
contrib/density/src/globals.h Normal file
View File

@ -0,0 +1,232 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 11/10/13 02:01
*/
#ifndef DENSITY_GLOBALS_H
#define DENSITY_GLOBALS_H
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <stddef.h>
#include "density_api.h"
#if defined(__clang__) || defined(__GNUC__)
#define DENSITY_FORCE_INLINE inline __attribute__((always_inline))
#define DENSITY_RESTRICT restrict
#define DENSITY_RESTRICT_DECLARE
#define DENSITY_MEMCPY __builtin_memcpy
#define DENSITY_MEMMOVE __builtin_memmove
#define DENSITY_MEMSET __builtin_memset
#define DENSITY_LIKELY(x) __builtin_expect(!!(x), 1)
#define DENSITY_UNLIKELY(x) __builtin_expect(!!(x), 0)
#define DENSITY_PREFETCH(x) __builtin_prefetch(x)
#define DENSITY_CTZ(x) __builtin_ctz(x)
#if defined(__BYTE_ORDER__)
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define DENSITY_LITTLE_ENDIAN
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#define DENSITY_BIG_ENDIAN
#else
#error Unsupported endianness
#endif
#else
#error Unkwnown endianness
#endif
#elif defined(_MSC_VER)
#include <string.h>
#include <intrin.h>
#define DENSITY_FORCE_INLINE __forceinline
#define DENSITY_RESTRICT __restrict
#define DENSITY_RESTRICT_DECLARE __restrict
#define DENSITY_MEMCPY memcpy
#define DENSITY_MEMMOVE memmove
#define DENSITY_MEMSET memset
#define DENSITY_LIKELY(x) (x)
#define DENSITY_UNLIKELY(x) (x)
#define DENSITY_PREFETCH(x) ((void)(x))
DENSITY_FORCE_INLINE uint_fast8_t density_msvc_ctz(uint64_t value) {
unsigned long trailing_zero = 0;
if (_BitScanForward(&trailing_zero, (unsigned long)value))
return (uint_fast8_t)trailing_zero;
else
return 0;
}
#define DENSITY_CTZ(x) density_msvc_ctz(x)
#define DENSITY_LITTLE_ENDIAN // Little endian by default on Windows
#else
#error Unsupported compiler
#endif
#ifdef DENSITY_LITTLE_ENDIAN
#define DENSITY_LITTLE_ENDIAN_64(b) ((uint64_t)b)
#define DENSITY_LITTLE_ENDIAN_32(b) ((uint32_t)b)
#define DENSITY_LITTLE_ENDIAN_16(b) ((uint16_t)b)
#elif defined(DENSITY_BIG_ENDIAN)
#if __GNUC__ * 100 + __GNUC_MINOR__ >= 403
#define DENSITY_LITTLE_ENDIAN_64(b) __builtin_bswap64(b)
#define DENSITY_LITTLE_ENDIAN_32(b) __builtin_bswap32(b)
#define DENSITY_LITTLE_ENDIAN_16(b) __builtin_bswap16(b)
#else
#warning Using bulk byte swap routines. Expect performance issues.
#define DENSITY_LITTLE_ENDIAN_64(b) ((((b) & 0xFF00000000000000ull) >> 56) | (((b) & 0x00FF000000000000ull) >> 40) | (((b) & 0x0000FF0000000000ull) >> 24) | (((b) & 0x000000FF00000000ull) >> 8) | (((b) & 0x00000000FF000000ull) << 8) | (((b) & 0x0000000000FF0000ull) << 24ull) | (((b) & 0x000000000000FF00ull) << 40) | (((b) & 0x00000000000000FFull) << 56))
#define DENSITY_LITTLE_ENDIAN_32(b) ((((b) & 0xFF000000) >> 24) | (((b) & 0x00FF0000) >> 8) | (((b) & 0x0000FF00) << 8) | (((b) & 0x000000FF) << 24))
#define DENSITY_LITTLE_ENDIAN_16(b) ((((b) & 0xFF00) >> 8) | (((b) & 0x00FF) << 8))
#endif
#else
#error Unsupported endianness
#endif
#define DENSITY_MAX_2(a, b) (((a)>(b))?(a):(b))
#define DENSITY_MAX_3(a, b, c) (DENSITY_MAX_2(DENSITY_MAX_2(a, b), c))
#define DENSITY_FORMAT(v) 0##v##llu
#define DENSITY_ISOLATE(b, p) ((DENSITY_FORMAT(b) / p) & 0x1)
#define DENSITY_BINARY_TO_UINT(b) ((DENSITY_ISOLATE(b, 1llu) ? 0x1 : 0)\
+ (DENSITY_ISOLATE(b, 8llu) ? 0x2 : 0)\
+ (DENSITY_ISOLATE(b, 64llu) ? 0x4 : 0)\
+ (DENSITY_ISOLATE(b, 512llu) ? 0x8 : 0)\
+ (DENSITY_ISOLATE(b, 4096llu) ? 0x10 : 0)\
+ (DENSITY_ISOLATE(b, 32768llu) ? 0x20 : 0)\
+ (DENSITY_ISOLATE(b, 262144llu) ? 0x40 : 0)\
+ (DENSITY_ISOLATE(b, 2097152llu) ? 0x80 : 0)\
+ (DENSITY_ISOLATE(b, 16777216llu) ? 0x100 : 0)\
+ (DENSITY_ISOLATE(b, 134217728llu) ? 0x200 : 0)\
+ (DENSITY_ISOLATE(b, 1073741824llu) ? 0x400 : 0)\
+ (DENSITY_ISOLATE(b, 8589934592llu) ? 0x800 : 0)\
+ (DENSITY_ISOLATE(b, 68719476736llu) ? 0x1000 : 0)\
+ (DENSITY_ISOLATE(b, 549755813888llu) ? 0x2000 : 0)\
+ (DENSITY_ISOLATE(b, 4398046511104llu) ? 0x4000 : 0)\
+ (DENSITY_ISOLATE(b, 35184372088832llu) ? 0x8000 : 0)\
+ (DENSITY_ISOLATE(b, 281474976710656llu) ? 0x10000 : 0)\
+ (DENSITY_ISOLATE(b, 2251799813685248llu) ? 0x20000 : 0))
#define DENSITY_UNROLL_2(op) op; op
#define DENSITY_UNROLL_4(op) DENSITY_UNROLL_2(op); DENSITY_UNROLL_2(op)
#define DENSITY_UNROLL_8(op) DENSITY_UNROLL_4(op); DENSITY_UNROLL_4(op)
#define DENSITY_UNROLL_16(op) DENSITY_UNROLL_8(op); DENSITY_UNROLL_8(op)
#define DENSITY_UNROLL_32(op) DENSITY_UNROLL_16(op); DENSITY_UNROLL_16(op)
#define DENSITY_UNROLL_64(op) DENSITY_UNROLL_32(op); DENSITY_UNROLL_32(op)
#define DENSITY_CASE_GENERATOR_2(op_a, flag_a, op_b, flag_b, op_mid, shift)\
case ((flag_b << shift) | flag_a):\
op_a;\
op_mid;\
op_b;\
break;
#define DENSITY_CASE_GENERATOR_4(op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\
case ((flag_d << (shift * 3)) | (flag_c << (shift * 2)) | (flag_b << shift) | flag_a):\
op_a;\
op_mid;\
op_b;\
op_mid;\
op_c;\
op_mid;\
op_d;\
break;
#define DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_1, flag_1, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\
DENSITY_CASE_GENERATOR_2(op_1, flag_1, op_a, flag_a, op_mid, shift);\
DENSITY_CASE_GENERATOR_2(op_1, flag_1, op_b, flag_b, op_mid, shift);\
DENSITY_CASE_GENERATOR_2(op_1, flag_1, op_c, flag_c, op_mid, shift);\
DENSITY_CASE_GENERATOR_2(op_1, flag_1, op_d, flag_d, op_mid, shift);
#define DENSITY_CASE_GENERATOR_4_2_COMBINED(op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\
DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_a, flag_a, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\
DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_b, flag_b, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\
DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_c, flag_c, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\
DENSITY_CASE_GENERATOR_4_2_LAST_1_COMBINED(op_d, flag_d, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);
#define DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\
DENSITY_CASE_GENERATOR_4(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_a, flag_a, op_mid, shift);\
DENSITY_CASE_GENERATOR_4(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_b, flag_b, op_mid, shift);\
DENSITY_CASE_GENERATOR_4(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_c, flag_c, op_mid, shift);\
DENSITY_CASE_GENERATOR_4(op_1, flag_1, op_2, flag_2, op_3, flag_3, op_d, flag_d, op_mid, shift);
#define DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_2, flag_2, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\
DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_a, flag_a, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\
DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_b, flag_b, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\
DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_c, flag_c, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\
DENSITY_CASE_GENERATOR_4_4_LAST_1_COMBINED(op_1, flag_1, op_2, flag_2, op_d, flag_d, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);
#define DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_1, flag_1, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\
DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_a, flag_a, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\
DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_b, flag_b, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\
DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_c, flag_c, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\
DENSITY_CASE_GENERATOR_4_4_LAST_2_COMBINED(op_1, flag_1, op_d, flag_d, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);
#define DENSITY_CASE_GENERATOR_4_4_COMBINED(op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift)\
DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_a, flag_a, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\
DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_b, flag_b, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\
DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_c, flag_c, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);\
DENSITY_CASE_GENERATOR_4_4_LAST_3_COMBINED(op_d, flag_d, op_a, flag_a, op_b, flag_b, op_c, flag_c, op_d, flag_d, op_mid, shift);
#define DENSITY_DICTIONARY_PREFERRED_RESET_CYCLE_SHIFT 6
#define DENSITY_DICTIONARY_PREFERRED_RESET_CYCLE (1 << DENSITY_DICTIONARY_PREFERRED_RESET_CYCLE_SHIFT)
#define density_bitsizeof(x) (8 * sizeof(x))
#define DENSITY_SPOOKYHASH_SEED_1 (0xabc)
#define DENSITY_SPOOKYHASH_SEED_2 (0xdef)
DENSITY_WINDOWS_EXPORT uint8_t density_version_major();
DENSITY_WINDOWS_EXPORT uint8_t density_version_minor();
DENSITY_WINDOWS_EXPORT uint8_t density_version_revision();
/**********************************************************************************************************************
* *
* Global compile-time switches *
* *
**********************************************************************************************************************/
#define DENSITY_MAJOR_VERSION 0
#define DENSITY_MINOR_VERSION 14
#define DENSITY_REVISION 2
#endif

57
contrib/density/src/structure/header.c Normal file
View File

@ -0,0 +1,57 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 11/10/13 17:56
*/
#include "header.h"
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_header_read(const uint8_t **DENSITY_RESTRICT in, density_header *DENSITY_RESTRICT header) {
header->version[0] = *(*in);
header->version[1] = *(*in + 1);
header->version[2] = *(*in + 2);
header->algorithm = *(*in + 3);
*in += sizeof(density_header);
}
DENSITY_WINDOWS_EXPORT DENSITY_FORCE_INLINE void density_header_write(uint8_t **DENSITY_RESTRICT out, const DENSITY_ALGORITHM algorithm) {
*(*out) = DENSITY_MAJOR_VERSION;
*(*out + 1) = DENSITY_MINOR_VERSION;
*(*out + 2) = DENSITY_REVISION;
*(*out + 3) = algorithm;
*(*out + 4) = 0;
*(*out + 5) = 0;
*(*out + 6) = 0;
*(*out + 7) = 0;
*out += sizeof(density_header);
}

58
contrib/density/src/structure/header.h Normal file
View File

@ -0,0 +1,58 @@
/*
* Density
*
* Copyright (c) 2013, Guillaume Voirin
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* 11/10/13 02:06
*/
#ifndef DENSITY_FILE_HEADER_H
#define DENSITY_FILE_HEADER_H
#include <stdio.h>
#include <time.h>
#include "../globals.h"
#include "../density_api.h"
#pragma pack(push)
#pragma pack(4)
typedef struct {
density_byte version[3];
density_byte algorithm;
density_byte reserved[4];
} density_header;
#pragma pack(pop)
DENSITY_WINDOWS_EXPORT void density_header_read(const uint8_t ** DENSITY_RESTRICT_DECLARE, density_header * DENSITY_RESTRICT_DECLARE);
DENSITY_WINDOWS_EXPORT void density_header_write(uint8_t ** DENSITY_RESTRICT_DECLARE, const DENSITY_ALGORITHM);
#endif

21
contrib/lizard/.gitattributes vendored Normal file
View File

@ -0,0 +1,21 @@
# Set the default behavior
* text eol=lf
# Explicitly declare source files
*.c text eol=lf
*.h text eol=lf
# Denote files that should not be modified.
*.odt binary
*.png binary
# Visual Studio
*.sln text eol=crlf
*.vcxproj* text eol=crlf
*.vcproj* text eol=crlf
*.suo binary
*.rc text eol=crlf
# Windows
*.bat text eol=crlf
*.cmd text eol=crlf

37
contrib/lizard/.gitignore vendored Normal file
View File

@ -0,0 +1,37 @@
# Object files
*.o
*.ko
# Libraries
*.lib
*.a
# Shared objects (inc. Windows DLLs)
*.dll
*.so
*.so.*
*.dylib
# Executables
*.exe
*.out
*.app
# IDE / editors files
*.suo
*.user
.clang_complete
# Directories
_lizardbench/
_visual/
_codelite/
_backup/
_codelite_lz4/
# Archives
*.zip
*.liz
*.txt
*.bat

268
contrib/lizard/.travis.yml Normal file
View File

@ -0,0 +1,268 @@
language: c
matrix:
fast_finish: true
include:
- name: macOS
os: osx
env: Ubu=OS_X_Mavericks Cmd='make -C tests test-lizard CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"' COMPILER=clang
# Ubuntu 18.04
- name: Ubuntu 18.04, gcc-9, 64-bit and 32-bit tests
dist: bionic
script:
- CC=gcc-9; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
- CC=gcc-9; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror
before_install:
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
- sudo apt-get -q update
- sudo apt-get --no-install-suggests --no-install-recommends -yq install gcc-multilib gcc-9 gcc-9-multilib
- name: Ubuntu 18.04, clang-9, 64-bit tests
dist: bionic
script:
- CC=clang-9; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- sourceline: 'deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main'
key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
packages:
- clang-9
# Ubuntu 16.04 gcc
- name: Ubuntu 16.04, gcc-9, 64-bit and 32-bit tests
dist: xenial
script:
- CC=gcc-9; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
- CC=gcc-9; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-multilib
- gcc-9
- gcc-9-multilib
- name: Ubuntu 16.04, gcc-8, 64-bit and 32-bit tests
dist: xenial
script:
- CC=gcc-8; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
- CC=gcc-8; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-multilib
- gcc-8
- gcc-8-multilib
- name: Ubuntu 16.04, gcc-7, 64-bit and 32-bit tests
dist: xenial
script:
- CC=gcc-7; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
- CC=gcc-7; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-multilib
- gcc-7
- gcc-7-multilib
- name: Ubuntu 16.04, gcc-6, 64-bit and 32-bit tests
dist: xenial
script:
- CC=gcc-6; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
- CC=gcc-6; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-multilib
- gcc-6
- gcc-6-multilib
- name: Ubuntu 16.04, gcc-5, 64-bit and 32-bit tests
dist: xenial
script:
- CC=gcc-5; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
- CC=gcc-5; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-multilib
- gcc-5
- gcc-5-multilib
- name: Ubuntu 16.04, gcc-4.8, 64-bit and 32-bit tests
dist: xenial
script:
- CC=gcc-4.8; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
- CC=gcc-4.8; $CC -v; make -C tests test-lizardc32 test-fullbench32 test-frametest32 test-fuzzer32 MOREFLAGS=-Werror
addons:
apt:
packages:
- gcc-multilib
- gcc-4.8
- gcc-4.8-multilib
# Ubuntu 16.04 clang
- name: Ubuntu 16.04, clang-9, 64-bit tests
dist: xenial
script:
- CC=clang-9; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- sourceline: 'deb https://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main'
key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
packages:
- clang-9
- name: Ubuntu 16.04, clang-8, 64-bit tests
dist: xenial
script:
- CC=clang-8; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-xenial-8
packages:
- clang-8
- name: Ubuntu 16.04, clang-7, 64-bit tests
dist: xenial
script:
- CC=clang-7; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-xenial-7
packages:
- clang-7
- name: Ubuntu 16.04, clang-6.0, 64-bit tests
dist: xenial
script:
- CC=clang-6.0; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-xenial-6.0
packages:
- clang-6.0
- name: Ubuntu 16.04, clang-5.0, 64-bit tests
dist: xenial
script:
- CC=clang-5.0; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-xenial-5.0
packages:
- clang-5.0
- name: Ubuntu 16.04, clang-4.0, 64-bit tests
dist: xenial
script:
- CC=clang-4.0; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-xenial-4.0
packages:
- clang-4.0
- name: Ubuntu 14.04, clang-3.8, 64-bit tests
dist: trusty
script:
- CC=clang-3.8; $CC -v; make -C tests test-lizard test-fullbench test-frametest test-fuzzer MOREFLAGS=-Werror
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-trusty-3.8
packages:
- clang-3.8
- name: Ubuntu 16.04, misc tests
env: Cmd="make c_standards && make gpptest-native && make clean && make examples && make clean && make cmake && make clean && make travis-install && make clean && make clangtest-native" COMPILER=cc
dist: xenial
- name: Ubuntu 14.04, arm and aarch64
env: Cmd='make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static && make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static' COMPILER=arm-linux-gnueabi-gcc
dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-arm
- qemu-user-static
- gcc-arm-linux-gnueabi
- libc6-dev-armel-cross
- gcc-aarch64-linux-gnu
- libc6-dev-arm64-cross
- name: Ubuntu 14.04, powerpc and ppc64
env: Cmd='make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64' COMPILER=powerpc-linux-gnu-gcc
dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-ppc
- qemu-user-static
- gcc-powerpc-linux-gnu
- name: Ubuntu 14.04, valgrind
env: Cmd='make -C tests test-mem MOREFLAGS=-DLIZARD_RESET_MEM' COMPILER=cc
dist: trusty
sudo: required
addons:
apt:
packages:
- valgrind
#- env: Ubu=14.04 Cmd='make sanitize' COMPILER=clang
# dist: trusty
# sudo: required
# addons:
# apt:
# packages:
# - valgrind
#- env: Ubu=14.04 Cmd='make staticAnalyze' COMPILER=clang
# dist: trusty
# sudo: required
# addons:
# apt:
# packages:
# - clang
script:
- echo Cmd=$Cmd
- $COMPILER -v
- sh -c "$Cmd"

15
contrib/lizard/LICENSE Normal file
View File

@ -0,0 +1,15 @@
This repository uses 2 different licenses :
- all files in the `lib` directory use a BSD 2-Clause license
- all other files use a GPLv2 license, unless explicitly stated otherwise
Relevant license is reminded at the top of each source file,
and with the presence of COPYING or LICENSE file.
This model emphasizes the fact that
only files in the `lib` directory are designed to be included into 3rd party projects.
Other files, such as those from `programs` or `examples` directory,
are not intended to be compiled outside of their context.
They can serve as source of inspiration,
but they should not be copy/pasted into 3rd party projects,
as this scenario is not supported.

41
contrib/lizard/NEWS Normal file
View File

@ -0,0 +1,41 @@
Lizard v1.0
- LZ5 v2.0 was renamed to Lizard v1.0
- improved compression speed with many small files
LZ5 v2.0
- this version is optimized for decompression speed (instead of ratio for previous versions)
- LZ5 v2.0 contains 4 compression methods:
fastLZ4 : compression levels -10...-19 are designed to give better decompression speed than LZ4 i.e. over 2000 MB/s
LIZv1 : compression levels -20...-29 are designed to give better ratio than LZ4 keeping 75% decompression speed
fastLZ4 + Huffman : compression levels -30...-39 add Huffman coding to fastLZ4
LIZv1 + Huffman : compression levels -40...-49 give the best ratio (comparable to zlib and low levels of zstd/brotli) at decompression speed of 1000 MB/s
LZ5 v1.5
- introduced compatibility with Visual C++ 2010 and newer
- attached Visual Studio 2010 project
- thoroughly tested with 21 Travis CI and 7 AppVeyor CI tests
- fixed bug with reusing a context in lizard_frame.c (LizardF_compressBegin and Lizard_compress_HC_continue)
- fixed rare bug in match finder (concerns levels 4 - 15)
LZ5 v1.4.1
- fixed bug with a backward match extension (level 11 and 12)
LZ5 v1.4
- improved: levels from 13 to 15 (maximum compression ratio)
- added a new parser: LizardHC_optimal_price_bt
- updated documentation: lizard_Block_format.md and lizard_Frame_format.md
- changed lizard.exe: the "-B" option with block size [1-7] = 64KB, 256KB, 1MB, 4MB, 16MB, 64MB, 256MB (default : 4 = 4MB)
LZ5 v1.3.3
- added: new levels from 11 to 18 (maximum compression ratio)
- added: a new parser: LizardHC_optimal_price
- fixed: buffer-overflow during decompression (thanks to m^2)
LZ5 r132
- improved compression ratio
- added: new parsers: LizardHC_fast, LizardHC_price_fast, LizardHC_lowest_price
- added: a special 1-byte codeword for the last occured offset
- added: support for 3-byte long matches (MINMATCH = 3)
LZ5 r131
The first release based on LZ4 r132 dev

95
contrib/lizard/README.md Normal file
View File

@ -0,0 +1,95 @@
Lizard - efficient compression with very fast decompression
--------------------------------------------------------
Lizard (formerly LZ5) is a lossless compression algorithm which contains 4 compression methods:
- fastLZ4 : compression levels -10...-19 are designed to give better decompression speed than [LZ4] i.e. over 2000 MB/s
- LIZv1 : compression levels -20...-29 are designed to give better ratio than [LZ4] keeping 75% decompression speed
- fastLZ4 + Huffman : compression levels -30...-39 add Huffman coding to fastLZ4
- LIZv1 + Huffman : compression levels -40...-49 give the best ratio (comparable to [zlib] and low levels of [zstd]/[brotli]) at decompression speed of 1000 MB/s
Lizard library is based on frequently used [LZ4] library by Yann Collet but the Lizard compression format is not compatible with LZ4.
Lizard library is provided as open-source software using BSD 2-Clause license.
The high compression/decompression speed is achieved without any SSE and AVX extensions.
|Branch |Status |
|------------|---------|
|lz5_v1.5 | [![Build Status][travis15Badge]][travisLink] [![Build status][Appveyor15Badge]][AppveyorLink] |
|lizard | [![Build Status][travis20Badge]][travisLink] [![Build status][Appveyor20Badge]][AppveyorLink] |
[travis15Badge]: https://travis-ci.org/inikep/lizard.svg?branch=lz5_v1.5 "Continuous Integration test suite"
[travis20Badge]: https://travis-ci.org/inikep/lizard.svg?branch=lizard "Continuous Integration test suite"
[travisLink]: https://travis-ci.org/inikep/lizard
[Appveyor15Badge]: https://ci.appveyor.com/api/projects/status/cqw7emcuqge369p0/branch/lz5_v1.5?svg=true "Visual test suite"
[Appveyor20Badge]: https://ci.appveyor.com/api/projects/status/cqw7emcuqge369p0/branch/lizard?svg=true "Visual test suite"
[AppveyorLink]: https://ci.appveyor.com/project/inikep/lizard
[LZ4]: https://github.com/lz4/lz4
[zlib]: https://github.com/madler/zlib
[zstd]: https://github.com/facebook/zstd
[brotli]: https://github.com/google/brotli
Benchmarks
-------------------------
The following results are obtained with [lzbench](https://github.com/inikep/lzbench) and `-t16,16`
using 1 core of Intel Core i5-4300U, Windows 10 64-bit (MinGW-w64 compilation under gcc 6.2.0)
with [silesia.tar] which contains tarred files from [Silesia compression corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia).
| Compressor name | Compression| Decompress.| Compr. size | Ratio |
| --------------- | -----------| -----------| ----------- | ----- |
| memcpy | 7332 MB/s | 8719 MB/s | 211947520 |100.00 |
| lz4 1.7.3 | 440 MB/s | 2318 MB/s | 100880800 | 47.60 |
| lz4hc 1.7.3 -1 | 98 MB/s | 2121 MB/s | 87591763 | 41.33 |
| lz4hc 1.7.3 -4 | 55 MB/s | 2259 MB/s | 79807909 | 37.65 |
| lz4hc 1.7.3 -9 | 22 MB/s | 2315 MB/s | 77892285 | 36.75 |
| lz4hc 1.7.3 -12 | 17 MB/s | 2323 MB/s | 77849762 | 36.73 |
| lz4hc 1.7.3 -16 | 10 MB/s | 2323 MB/s | 77841782 | 36.73 |
| lizard 1.0 -10 | 346 MB/s | 2610 MB/s | 103402971 | 48.79 |
| lizard 1.0 -12 | 103 MB/s | 2458 MB/s | 86232422 | 40.69 |
| lizard 1.0 -15 | 50 MB/s | 2552 MB/s | 81187330 | 38.31 |
| lizard 1.0 -19 | 3.04 MB/s | 2497 MB/s | 77416400 | 36.53 |
| lizard 1.0 -21 | 157 MB/s | 1795 MB/s | 89239174 | 42.10 |
| lizard 1.0 -23 | 30 MB/s | 1778 MB/s | 81097176 | 38.26 |
| lizard 1.0 -26 | 6.63 MB/s | 1734 MB/s | 74503695 | 35.15 |
| lizard 1.0 -29 | 1.37 MB/s | 1634 MB/s | 68694227 | 32.41 |
| lizard 1.0 -30 | 246 MB/s | 909 MB/s | 85727429 | 40.45 |
| lizard 1.0 -32 | 94 MB/s | 1244 MB/s | 76929454 | 36.30 |
| lizard 1.0 -35 | 47 MB/s | 1435 MB/s | 73850400 | 34.84 |
| lizard 1.0 -39 | 2.94 MB/s | 1502 MB/s | 69807522 | 32.94 |
| lizard 1.0 -41 | 126 MB/s | 961 MB/s | 76100661 | 35.91 |
| lizard 1.0 -43 | 28 MB/s | 1101 MB/s | 70955653 | 33.48 |
| lizard 1.0 -46 | 6.25 MB/s | 1073 MB/s | 65413061 | 30.86 |
| lizard 1.0 -49 | 1.27 MB/s | 1064 MB/s | 60679215 | 28.63 |
| zlib 1.2.8 -1 | 66 MB/s | 244 MB/s | 77259029 | 36.45 |
| zlib 1.2.8 -6 | 20 MB/s | 263 MB/s | 68228431 | 32.19 |
| zlib 1.2.8 -9 | 8.37 MB/s | 266 MB/s | 67644548 | 31.92 |
| zstd 1.1.1 -1 | 235 MB/s | 645 MB/s | 73659468 | 34.75 |
| zstd 1.1.1 -2 | 181 MB/s | 600 MB/s | 70168955 | 33.11 |
| zstd 1.1.1 -5 | 88 MB/s | 565 MB/s | 65002208 | 30.67 |
| zstd 1.1.1 -8 | 31 MB/s | 619 MB/s | 61026497 | 28.79 |
| zstd 1.1.1 -11 | 16 MB/s | 613 MB/s | 59523167 | 28.08 |
| zstd 1.1.1 -15 | 4.97 MB/s | 639 MB/s | 58007773 | 27.37 |
| zstd 1.1.1 -18 | 2.87 MB/s | 583 MB/s | 55294241 | 26.09 |
| zstd 1.1.1 -22 | 1.44 MB/s | 505 MB/s | 52731930 | 24.88 |
| brotli 0.5.2 -0 | 217 MB/s | 244 MB/s | 78226979 | 36.91 |
| brotli 0.5.2 -2 | 96 MB/s | 283 MB/s | 68066621 | 32.11 |
| brotli 0.5.2 -5 | 24 MB/s | 312 MB/s | 60801716 | 28.69 |
| brotli 0.5.2 -8 | 5.56 MB/s | 324 MB/s | 57382470 | 27.07 |
| brotli 0.5.2 -11 | 0.39 MB/s | 266 MB/s | 51138054 | 24.13 |
[silesia.tar]: https://drive.google.com/file/d/0BwX7dtyRLxThenZpYU9zLTZhR1k/view?usp=sharing
Documentation
-------------------------
The raw Lizard block compression format is detailed within [lizard_Block_format].
To compress an arbitrarily long file or data stream, multiple blocks are required.
Organizing these blocks and providing a common header format to handle their content
is the purpose of the Frame format, defined into [lizard_Frame_format].
Interoperable versions of Lizard must respect this frame format.
[lizard_Block_format]: doc/lizard_Block_format.md
[lizard_Frame_format]: doc/lizard_Frame_format.md

143
contrib/lizard/appveyor.yml Normal file
View File

@ -0,0 +1,143 @@
version: 1.0.{build}
environment:
matrix:
- COMPILER: "visual"
CONFIGURATION: "Debug"
PLATFORM: "x64"
- COMPILER: "visual"
CONFIGURATION: "Debug"
PLATFORM: "Win32"
- COMPILER: "visual"
CONFIGURATION: "Release"
PLATFORM: "x64"
- COMPILER: "visual"
CONFIGURATION: "Release"
PLATFORM: "Win32"
- COMPILER: "gcc"
PLATFORM: "mingw64"
- COMPILER: "gcc"
PLATFORM: "mingw32"
- COMPILER: "gcc"
PLATFORM: "clang"
install:
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
- MKDIR bin
- if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH%
- if [%COMPILER%]==[gcc] (
SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe &&
COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe
) else (
IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;")
)
build_script:
- if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL%
- if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
- if [%PLATFORM%]==[clang] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
- ECHO *** &&
ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% &&
ECHO ***
- if [%PLATFORM%]==[clang] (clang -v)
- if [%COMPILER%]==[gcc] (gcc -v)
- if [%COMPILER%]==[gcc] (
echo ----- &&
make -v &&
echo ----- &&
if not [%PLATFORM%]==[clang] (
make -C programs lizard && make -C tests fullbench && make -C lib lib
) ELSE (
make -C programs lizard CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" &&
make -C tests fullbench CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" &&
make -C lib lib CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"
)
)
- if [%COMPILER%]==[gccX] if not [%PLATFORM%]==[clang] (
MKDIR bin\dll bin\static bin\example bin\include &&
COPY tests\fullbench.c bin\example\ &&
COPY lib\xxhash.c bin\example\ &&
COPY lib\xxhash.h bin\example\ &&
COPY lib\lizard.h bin\include\ &&
COPY lib\lizardhc.h bin\include\ &&
COPY lib\lizard_frame.h bin\include\ &&
COPY lib\liblizard.a bin\static\liblizard_static.lib &&
COPY lib\dll\liblizard.* bin\dll\ &&
COPY lib\dll\example\Makefile bin\example\ &&
COPY lib\dll\example\fullbench-dll.* bin\example\ &&
COPY lib\dll\example\README.md bin\ &&
COPY programs\lizard.exe bin\lizard.exe
)
- if [%COMPILER%]==[gccX] if [%PLATFORM%]==[mingw64] (
7z.exe a bin\lizard_x64.zip NEWS .\bin\lizard.exe .\bin\README.md .\bin\example .\bin\dll .\bin\static .\bin\include &&
appveyor PushArtifact bin\lizard_x64.zip
)
- if [%COMPILER%]==[gccX] if [%PLATFORM%]==[mingw32] (
7z.exe a bin\lizard_x86.zip NEWS .\bin\lizard.exe .\bin\README.md .\bin\example .\bin\dll .\bin\static .\bin\include &&
appveyor PushArtifact bin\lizard_x86.zip
)
- if [%COMPILER%]==[gcc] (COPY tests\fullbench.exe programs\)
- if [%COMPILER%]==[visual] (
ECHO *** &&
ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lizard.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
ECHO *** &&
ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lizard.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
ECHO *** &&
ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lizard.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
ECHO *** &&
ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lizard.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
COPY visual\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe programs\
)
test_script:
- ECHO *** &&
ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% &&
ECHO ***
- if not [%COMPILER%]==[unknown] (
CD programs &&
lizard -h &&
lizard -i1b lizard.exe &&
lizard -i1b5 lizard.exe &&
lizard -i1b10 lizard.exe &&
lizard -i1b15 lizard.exe &&
echo ------- lizard tested ------- &&
fullbench.exe -i1 fullbench.exe
)
artifacts:
- path: bin\lizard_x64.zip
- path: bin\lizard_x86.zip
deploy:
- provider: GitHub
artifact: bin\lizard_x64.zip
auth_token:
secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4
force_update: true
prerelease: true
on:
COMPILER: gcc
PLATFORM: "mingw64"
appveyor_repo_tag: true
branch: autobuild
- provider: GitHub
artifact: bin\lizard_x86.zip
auth_token:
secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4
force_update: true
prerelease: true
on:
COMPILER: gcc
PLATFORM: "mingw32"
appveyor_repo_tag: true
branch: autobuild

24
contrib/lizard/contrib/djgpp/LICENSE Normal file
View File

@ -0,0 +1,24 @@
Copyright (c) 2014, lpsantil
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

21
contrib/lizard/contrib/djgpp/README.MD Normal file
View File

@ -0,0 +1,21 @@
# lizard for DOS/djgpp
This file details on how to compile lizard.exe, and liblizard.a for use on DOS/djgpp using
Andrew Wu's build-djgpp cross compilers ([GH][0], [Binaries][1]) on OSX, Linux.
## Setup
* Download a djgpp tarball [binaries][1] for your platform.
* Extract and install it (`tar jxvf djgpp-linux64-gcc492.tar.bz2`). Note the path. We'll assume `/home/user/djgpp`.
* Add the `bin` folder to your `PATH`. In bash, do `export PATH=/home/user/djgpp/bin:$PATH`.
* The `Makefile` in `contrib/djgpp/` sets up `CC`, `AR`, `LD` for you. So, `CC=i586-pc-msdosdjgpp-gcc`, `AR=i586-pc-msdosdjgpp-ar`, `LD=i586-pc-msdosdjgpp-gcc`.
## Building Lizard for DOS
In the base dir of lizard and with `contrib/djgpp/Makefile`, try:
Try:
* `make -f contrib/djgpp/Makefile`
* `make -f contrib/djgpp/Makefile liblizard.a`
* `make -f contrib/djgpp/Makefile lizard.exe`
* `make -f contrib/djgpp/Makefile DESTDIR=/home/user/dos install`, however it doesn't make much sense on a \*nix.
* You can also do `make -f contrib/djgpp/Makefile uninstall`
[0]: https://github.com/andrewwutw/build-djgpp
[1]: https://github.com/andrewwutw/build-djgpp/releases

181
contrib/lizard/doc/lizard_Block_format.md Normal file
View File

@ -0,0 +1,181 @@
Lizard v1.x Block Format Description
============================
Last revised: 2016-10-08
Authors : Yann Collet, Przemyslaw Skibinski
This specification is intended for developers
willing to produce Lizard-compatible compressed data blocks
using any programming language.
Lizard is an LZ77-type compressor with a fixed, byte-oriented encoding.
There is no framing layer as it is assumed to be handled by other parts of the system (see [Lizard Frame format]).
This design is assumed to favor simplicity and speed.
It helps later on for optimizations, compactness, and features.
This document describes only the block format,
not how the compressor nor decompressor actually work.
The correctness of the decompressor should not depend
on implementation details of the compressor, and vice versa.
[Lizard Frame format]: lizard_Frame_format.md
Division into blocks
--------------------
The input data is divided into blocks of maximum size LIZARD_BLOCK_SIZE (which is 128 KB). The subsequent blocks use the same sliding window and are dependent on previous blocks.
Our impementation of Lizard compressor divides input data into blocks of size of LIZARD_BLOCK_SIZE except the last one which usually will be smaller.
The output data is a single byte 'Compression_Level' and one or more blocks in the format described below.
Block header format
-----------------------
The block header is a single byte `Header_Byte` that is combination of following flags:
| Name | Value |
| --------------------- | --- |
| LIZARD_FLAG_LITERALS | 1 |
| LIZARD_FLAG_FLAGS | 2 |
| LIZARD_FLAG_OFF16LEN | 4 |
| LIZARD_FLAG_OFF24LEN | 8 |
| LIZARD_FLAG_LEN | 16 |
| LIZARD_FLAG_UNCOMPRESSED | 128 |
When `Header_Byte & LIZARD_FLAG_UNCOMPRESSED` is true then the block is followed by 3-byte `Uncompressed_length` and uncompressed data of given size.
Compressed block content
------------------------
When `Header_Byte & LIZARD_FLAG_UNCOMPRESSED` is false then compressed block contains of 5 streams:
- `Lengths_Stream` (compressed with Huffman if LIZARD_FLAG_LEN is set)
- `16-bit_Offsets_Stream` (compressed with Huffman if LIZARD_FLAG_OFF16LEN is set)
- `24-bit_Offsets_Stream` (compressed with Huffman if LIZARD_FLAG_OFF24LEN is set)
- `Tokens_Stream` (compressed with Huffman if LIZARD_FLAG_FLAGS is set)
- `Literals_Stream` (compressed with Huffman if LIZARD_FLAG_LITERALS is set)
Stream format
-------------
The single stream is either:
- if LIZARD_FLAG_XXX is not set: 3 byte `Stream_Length` followed by a given number bytes
- if LIZARD_FLAG_XXX is set: 3 byte `Original_Stream_Length`, 3 byte `Compressed_Stream_Length`, followed by a given number of Huffman compressed bytes
Lizard block decompression
-----------------------
At the beginning we have 5 streams and their sizes.
Decompressor should iterate through `Tokens_Stream`. Each token is 1-byte long and describes how to get data from other streams.
If token points a stream that is already empty it means that data is corrupted.
Lizard token decompression
-----------------------
The token is a one byte. Token decribes:
- how many literals should be copied from `Literals_Stream`
- if offset should be read from `16-bit_Offsets_Stream` or `24-bit_Offsets_Stream`
- how many bytes are part of a match and should be copied from a sliding window
Lizard uses 4 types of tokens:
- [0_MMMM_LLL] - 3-bit literal length (0-7+), use offset from `16-bit_Offsets_Stream`, 4-bit match length (4-15+)
- [1_MMMM_LLL] - 3-bit literal length (0-7+), use last offset, 4-bit match length (0-15+)
- token 31 - no literal length, use offset from `24-bit_Offsets_Stream`, match length (47+)
- token 0-30 - no literal length, use offset from `24-bit_Offsets_Stream`, 31 match lengths (16-46)
Lizard uses different output codewords and is not compatible with LZ4. LZ4 output codewords are 3 byte long (24-bit) and look as follows:
- LLLL_MMMM OOOOOOOO OOOOOOOO - 16-bit offset, 4-bit match length, 4-bit literal length
The format of `Lengths_Stream`
------------------------------
`Lengths_Stream` contains lenghts in the the following format:
- when 'First_Byte' is < 254 then lenght is equal 'First_Byte'
- when 'First_Byte' is 254 then lenght is equal to value of 2-bytes after 'First_Byte' i.e. 0-65536
- when 'First_Byte' is 255 then lenght is equal to value of 3-bytes after 'First_Byte' i.e. 0-16777215
[0_MMMM_LLL] and [1_MMMM_LLL] tokens
---------------------------------------
The length of literals to be copied from `Literals_Stream` depends on the literal length field (LLL) that uses 3 bits of the token.
Therefore each field ranges from 0 to 7.
If the value is 7, then the lenght is increased with a length taken from `Lengths_Stream`.
Example 1 : A literal length of 48 will be represented as :
- 7 : value for the 3-bits LLL field
- 41 : (=48-7) remaining length to reach 48 (in `Lengths_Stream`)
Example 2 : A literal length of 280 for will be represented as :
- 7 : value for the 3-bits LLL field
- 254 : informs that remaining length (=280-7) must be represented as 2-bytes (in `Lengths_Stream`)
- 273 : (=280-7) encoded as 2-bytes (in `Lengths_Stream`)
Example 3 : A literal length of 7 for will be represented as :
- 7 : value for the 3-bits LLL field
- 0 : (=7-7) yes, the zero must be output (in `Lengths_Stream`)
After copying 0 or more literals from `Literals_Stream` we can prepare the match copy operation which depends on a offset and a match length.
The flag "0" informs that decoder should use the last encoded offset.
The flag "1" informs that the offset is a 2 bytes value (16-bit), in little endian format and should be taken from `16-bit_Offsets_Stream`.
The match length depends on the match length field (MMMM) that uses 4 bits of the token.
Therefore each field ranges from 0 to 15. Values from 0-3 are forbidden with offset taken from `16-bit_Offsets_Stream`.
If the value is 15, then the lenght is increased with a length taken from `Lengths_Stream`.
With the offset and the match length,
the decoder can now proceed to copy the data from the already decoded buffer.
Lizard block epilogue
------------------
When all tokens are read from `Tokens_Stream` and interpreted all remaining streams should also be empty.
Otherwise, it means that the data is corrupted. The only exception is `Literals_Stream` that should have at least 16 remaining literals what
allows fast memory copy operations. The remaining literals up to the end of `Literals_Stream` should be appended to the output data.
Tokens 0-31
-----------
The offset is a 3 bytes value (24-bit), in little endian format and should be taken from `24-bit_Offsets_Stream`.
The offset represents the position of the match to be copied from.
1 means "current position - 1 byte".
The maximum offset value is (1<<24)-1, 1<<24 cannot be coded.
Note that 0 is an invalid value, not used.
The 'Token_Value' ranges from 0 to 31.
The match length is equal to 'Token_Value + 16 that is from 16 to 47.
If match length is 47, the lenght is increased with a length taken from `Lengths_Stream`.
Parsing restrictions
-----------------------
There are specific parsing rules to respect in order to remain compatible
with assumptions made by the decoder :
1. The last 16 bytes are always literals what allows fast memory copy operations.
2. The last match must start at least 20 bytes before end of block.
Consequently, a block with less than 20 bytes cannot be compressed.
These rules are in place to ensure that the decoder
will never read beyond the input buffer, nor write beyond the output buffer.
Note that the last sequence is also incomplete,
and stops right after literals.
Additional notes
-----------------------
There is no assumption nor limits to the way the compressor
searches and selects matches within the source data block.
It could be a fast scan, a multi-probe, a full search using BST,
standard hash chains or MMC, well whatever.
Advanced parsing strategies can also be implemented, such as lazy match,
or full optimal parsing.
All these trade-off offer distinctive speed/memory/compression advantages.
Whatever the method used by the compressor, its result will be decodable
by any Lizard decoder if it follows the format specification described above.

312
contrib/lizard/doc/lizard_Frame_format.md Normal file
View File

@ -0,0 +1,312 @@
Lizard v1.x Frame Format Description
=================================
###Notices
Copyright (c) 2013-2015 Yann Collet
Copyright (c) 2016 Przemyslaw Skibinski
Permission is granted to copy and distribute this document
for any purpose and without charge,
including translations into other languages
and incorporation into compilations,
provided that the copyright notice and this notice are preserved,
and that any substantive changes or deletions from the original
are clearly marked.
Distribution of this document is unlimited.
###Version
1.0 (8-10-2016)
Introduction
------------
The purpose of this document is to define a lossless compressed data format,
that is independent of CPU type, operating system,
file system and character set, suitable for
File compression, Pipe and streaming compression
using the Lizard algorithm.
The data can be produced or consumed,
even for an arbitrarily long sequentially presented input data stream,
using only an a priori bounded amount of intermediate storage,
and hence can be used in data communications.
The format uses the Lizard compression method,
and optional [xxHash-32 checksum method](https://github.com/Cyan4973/xxHash),
for detection of data corruption.
The data format defined by this specification
does not attempt to allow random access to compressed data.
This specification is intended for use by implementers of software
to compress data into Lizard format and/or decompress data from Lizard format.
The text of the specification assumes a basic background in programming
at the level of bits and other primitive data representations.
Unless otherwise indicated below,
a compliant compressor must produce data sets
that conform to the specifications presented here.
It doesn’t need to support all options though.
A compliant decompressor must be able to decompress
at least one working set of parameters
that conforms to the specifications presented here.
It may also ignore checksums.
Whenever it does not support a specific parameter within the compressed stream,
it must produce a non-ambiguous error code
and associated error message explaining which parameter is unsupported.
General Structure of Lizard Frame format
-------------------------------------
| MagicNb | F. Descriptor | Block | (...) | EndMark | C. Checksum |
|:-------:|:-------------:| ----- | ----- | ------- | ----------- |
| 4 bytes | 3-11 bytes | | | 4 bytes | 0-4 bytes |
__Magic Number__
4 Bytes, Little endian format.
Value : 0x184D2206 (it was 0x184D2204 for LZ4 and 0x184D2205 for LZ5 v1.x)
__Frame Descriptor__
3 to 11 Bytes, to be detailed in the next part.
Most important part of the spec.
__Data Blocks__
To be detailed later on.
That’s where compressed data is stored.
__EndMark__
The flow of blocks ends when the last data block has a size of “0”.
The size is expressed as a 32-bits value.
__Content Checksum__
Content Checksum verify that the full content has been decoded correctly.
The content checksum is the result
of [xxh32() hash function](https://github.com/Cyan4973/xxHash)
digesting the original (decoded) data as input, and a seed of zero.
Content checksum is only present when its associated flag
is set in the frame descriptor.
Content Checksum validates the result,
that all blocks were fully transmitted in the correct order and without error,
and also that the encoding/decoding process itself generated no distortion.
Its usage is recommended.
__Frame Concatenation__
In some circumstances, it may be preferable to append multiple frames,
for example in order to add new data to an existing compressed file
without re-framing it.
In such case, each frame has its own set of descriptor flags.
Each frame is considered independent.
The only relation between frames is their sequential order.
The ability to decode multiple concatenated frames
within a single stream or file
is left outside of this specification.
As an example, the reference lizard command line utility behavior is
to decode all concatenated frames in their sequential order.
Frame Descriptor
----------------
| FLG | BD | (Content Size) | HC |
| ------- | ------- |:--------------:| ------- |
| 1 byte | 1 byte | 0 - 8 bytes | 1 byte |
The descriptor uses a minimum of 3 bytes,
and up to 11 bytes depending on optional parameters.
__FLG byte__
| BitNb | 7-6 | 5 | 4 | 3 | 2 | 1-0 |
| ------- | ------- | ------- | --------- | ------- | --------- | -------- |
|FieldName| Version | B.Indep | B.Checksum| C.Size | C.Checksum|*Reserved*|
__BD byte__
| BitNb | 7 | 6-5-4 | 3-2-1-0 |
| ------- | -------- | ------------ | -------- |
|FieldName|*Reserved*| Block MaxSize|*Reserved*|
In the tables, bit 7 is highest bit, while bit 0 is lowest.
__Version Number__
2-bits field, must be set to “01”.
Any other value cannot be decoded by this version of the specification.
Other version numbers will use different flag layouts.
__Block Independence flag__
If this flag is set to “1”, blocks are independent.
If this flag is set to “0”, each block depends on previous ones
(up to Lizard window size, which is 16 MB).
In such case, it’s necessary to decode all blocks in sequence.
Block dependency improves compression ratio, especially for small blocks.
On the other hand, it makes direct jumps or multi-threaded decoding impossible.
__Block checksum flag__
If this flag is set, each data block will be followed by a 4-bytes checksum,
calculated by using the xxHash-32 algorithm on the raw (compressed) data block.
The intention is to detect data corruption (storage or transmission errors)
immediately, before decoding.
Block checksum usage is optional.
__Content Size flag__
If this flag is set, the uncompressed size of data included within the frame
will be present as an 8 bytes unsigned little endian value, after the flags.
Content Size usage is optional.
__Content checksum flag__
If this flag is set, a content checksum will be appended after the EndMark.
Recommended value : “1” (content checksum is present)
__Block Maximum Size__
This information is intended to help the decoder allocate memory.
Size here refers to the original (uncompressed) data size.
Block Maximum Size is one value among the following table :
| 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
| --- | ------ | ------ | ---- | ---- | ----- | ----- | ------ |
| N/A | 128 KB | 256 KB | 1 MB | 4 MB | 16 MB | 64 MB | 256 MB |
The decoder may refuse to allocate block sizes above a (system-specific) size.
Unused values may be used in a future revision of the spec.
A decoder conformant to the current version of the spec
is only able to decode blocksizes defined in this spec.
__Reserved bits__
Value of reserved bits **must** be 0 (zero).
Reserved bit might be used in a future version of the specification,
typically enabling new optional features.
If this happens, a decoder respecting the current version of the specification
shall not be able to decode such a frame.
__Content Size__
This is the original (uncompressed) size.
This information is optional, and only present if the associated flag is set.
Content size is provided using unsigned 8 Bytes, for a maximum of 16 HexaBytes.
Format is Little endian.
This value is informational, typically for display or memory allocation.
It can be skipped by a decoder, or used to validate content correctness.
__Header Checksum__
One-byte checksum of combined descriptor fields, including optional ones.
The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF `
using zero as a seed,
and the full Frame Descriptor as an input
(including optional fields when they are present).
A wrong checksum indicates an error in the descriptor.
Header checksum is informational and can be skipped.
Data Blocks
-----------
| Block Size | data | (Block Checksum) |
|:----------:| ------ |:----------------:|
| 4 bytes | | 0 - 4 bytes |
__Block Size__
This field uses 4-bytes, format is little-endian.
The highest bit is “1” if data in the block is uncompressed.
The highest bit is “0” if data in the block is compressed by Lizard.
All other bits give the size, in bytes, of the following data block
(the size does not include the block checksum if present).
Block Size shall never be larger than Block Maximum Size.
Such a thing could happen for incompressible source data.
In such case, such a data block shall be passed in uncompressed format.
__Data__
Where the actual data to decode stands.
It might be compressed or not, depending on previous field indications.
Uncompressed size of Data can be any size, up to “block maximum size”.
Note that data block is not necessarily full :
an arbitrary “flush” may happen anytime. Any block can be “partially filled”.
__Block checksum__
Only present if the associated flag is set.
This is a 4-bytes checksum value, in little endian format,
calculated by using the xxHash-32 algorithm on the raw (undecoded) data block,
and a seed of zero.
The intention is to detect data corruption (storage or transmission errors)
before decoding.
Block checksum is cumulative with Content checksum.
Skippable Frames
----------------
| Magic Number | Frame Size | User Data |
|:------------:|:----------:| --------- |
| 4 bytes | 4 bytes | |
Skippable frames allow the integration of user-defined data
into a flow of concatenated frames.
Its design is pretty straightforward,
with the sole objective to allow the decoder to quickly skip
over user-defined data and continue decoding.
For the purpose of facilitating identification,
it is discouraged to start a flow of concatenated frames with a skippable frame.
If there is a need to start such a flow with some user data
encapsulated into a skippable frame,
it’s recommended to start with a zero-byte Lizard frame
followed by a skippable frame.
This will make it easier for file type identifiers.
__Magic Number__
4 Bytes, Little endian format.
Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F.
All 16 values are valid to identify a skippable frame.
__Frame Size__
This is the size, in bytes, of the following User Data
(without including the magic number nor the size field itself).
4 Bytes, Little endian format, unsigned 32-bits.
This means User Data can’t be bigger than (2^32-1) Bytes.
__User Data__
User Data can be anything. Data will just be skipped by the decoder.
Version changes
---------------
1.0 : based on LZ4 Frame Format Description 1.5.1 (31/03/2015)

8
contrib/lizard/examples/.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
/Makefile.liz*
/printVersion
/doubleBuffer
/ringBuffer
/ringBufferHC
/lineCompress
/frameCompress
/*.exe

241
contrib/lizard/examples/HCStreaming_ringBuffer.c Normal file
View File

@ -0,0 +1,241 @@
// Lizard streaming API example : ring buffer
// Based on previous work from Takayuki Matsuoka
/**************************************
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS // for MSVC
# define snprintf sprintf_s
#endif
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#ifdef __GNUC__
# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
#endif
/**************************************
* Includes
**************************************/
#include "lizard_compress.h"
#include "lizard_decompress.h"
#include "lizard_common.h"
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
enum {
MESSAGE_MAX_BYTES = 1024,
RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES,
DEC_BUFFER_BYTES = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES // Intentionally larger to test unsynchronized ring buffers
};
size_t write_int32(FILE* fp, int32_t i) {
return fwrite(&i, sizeof(i), 1, fp);
}
size_t write_bin(FILE* fp, const void* array, int arrayBytes) {
return fwrite(array, 1, arrayBytes, fp);
}
size_t read_int32(FILE* fp, int32_t* i) {
return fread(i, sizeof(*i), 1, fp);
}
size_t read_bin(FILE* fp, void* array, int arrayBytes) {
return fread(array, 1, arrayBytes, fp);
}
void test_compress(FILE* outFp, FILE* inpFp)
{
Lizard_stream_t* lizardStream = Lizard_createStream(0);
static char inpBuf[RING_BUFFER_BYTES];
int inpOffset = 0;
if (!lizardStream) return;
for(;;)
{
// Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer.
char* const inpPtr = &inpBuf[inpOffset];
const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1;
const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
if (0 == inpBytes) break;
{
char cmpBuf[LIZARD_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
const int cmpBytes = Lizard_compress_continue(lizardStream, inpPtr, cmpBuf, inpBytes, Lizard_compressBound(inpBytes));
if(cmpBytes <= 0) break;
write_int32(outFp, cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
inpOffset += inpBytes;
// Wraparound the ringbuffer offset
if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES)
inpOffset = 0;
}
}
write_int32(outFp, 0);
Lizard_freeStream(lizardStream);
}
void test_decompress(FILE* outFp, FILE* inpFp)
{
static char decBuf[DEC_BUFFER_BYTES];
int decOffset = 0;
Lizard_streamDecode_t lizardStreamDecode_body = { 0 };
Lizard_streamDecode_t* lizardStreamDecode = &lizardStreamDecode_body;
for(;;)
{
int cmpBytes = 0;
char cmpBuf[LIZARD_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
{
const size_t r0 = read_int32(inpFp, &cmpBytes);
size_t r1;
if(r0 != 1 || cmpBytes <= 0)
break;
r1 = read_bin(inpFp, cmpBuf, cmpBytes);
if(r1 != (size_t) cmpBytes)
break;
}
{
char* const decPtr = &decBuf[decOffset];
const int decBytes = Lizard_decompress_safe_continue(
lizardStreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
if(decBytes <= 0)
break;
decOffset += decBytes;
write_bin(outFp, decPtr, decBytes);
// Wraparound the ringbuffer offset
if(decOffset >= DEC_BUFFER_BYTES - MESSAGE_MAX_BYTES)
decOffset = 0;
}
}
}
// Compare 2 files content
// return 0 if identical
// return ByteNb>0 if different
size_t compare(FILE* f0, FILE* f1)
{
size_t result = 1;
for (;;)
{
char b0[65536];
char b1[65536];
const size_t r0 = fread(b0, 1, sizeof(b0), f0);
const size_t r1 = fread(b1, 1, sizeof(b1), f1);
if ((r0==0) && (r1==0)) return 0; // success
if (r0 != r1)
{
size_t smallest = r0;
if (r1<r0) smallest = r1;
result += smallest;
break;
}
if (memcmp(b0, b1, r0))
{
unsigned errorPos = 0;
while ((errorPos < r0) && (b0[errorPos]==b1[errorPos])) errorPos++;
result += errorPos;
break;
}
result += sizeof(b0);
}
return result;
}
int main(int argc, char** argv)
{
char inpFilename[256] = { 0 };
char lizardFilename[256] = { 0 };
char decFilename[256] = { 0 };
unsigned fileID = 1;
unsigned pause = 0;
if(argc < 2) {
printf("Please specify input filename\n");
return 0;
}
if (!strcmp(argv[1], "-p")) pause = 1, fileID = 2;
snprintf(inpFilename, 256, "%s", argv[fileID]);
snprintf(lizardFilename, 256, "%s.lizs-%d", argv[fileID], 9);
snprintf(decFilename, 256, "%s.lizs-%d.dec", argv[fileID], 9);
printf("input = [%s]\n", inpFilename);
printf("lizard = [%s]\n", lizardFilename);
printf("decoded = [%s]\n", decFilename);
// compress
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* outFp = fopen(lizardFilename, "wb");
test_compress(outFp, inpFp);
fclose(outFp);
fclose(inpFp);
}
// decompress
{
FILE* inpFp = fopen(lizardFilename, "rb");
FILE* outFp = fopen(decFilename, "wb");
test_decompress(outFp, inpFp);
fclose(outFp);
fclose(inpFp);
}
// verify
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* decFp = fopen(decFilename, "rb");
const size_t cmp = compare(inpFp, decFp);
if(0 == cmp) {
printf("Verify : OK\n");
} else {
printf("Verify : NG : error at pos %u\n", (unsigned)cmp-1);
}
fclose(decFp);
fclose(inpFp);
}
if (pause)
{
printf("Press enter to continue ...\n");
getchar();
}
return 0;
}

8
contrib/lizard/examples/README.md Normal file
View File

@ -0,0 +1,8 @@
# Lizard examples
## Documents
- [Streaming API Basics](streaming_api_basics.md)
- Examples
- [Double Buffer](blockStreaming_doubleBuffer.md)
- [Line by Line Text Compression](blockStreaming_lineByLine.md)

203
contrib/lizard/examples/blockStreaming_doubleBuffer.c Normal file
View File

@ -0,0 +1,203 @@
// Lizard streaming API example : double buffer
// Copyright : Takayuki Matsuoka
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif
#include "lizard_common.h"
#include "lizard_decompress.h"
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
enum {
BLOCK_BYTES = 1024 * 8,
// BLOCK_BYTES = 1024 * 64,
};
size_t write_int(FILE* fp, int i) {
return fwrite(&i, sizeof(i), 1, fp);
}
size_t write_bin(FILE* fp, const void* array, size_t arrayBytes) {
return fwrite(array, 1, arrayBytes, fp);
}
size_t read_int(FILE* fp, int* i) {
return fread(i, sizeof(*i), 1, fp);
}
size_t read_bin(FILE* fp, void* array, size_t arrayBytes) {
return fread(array, 1, arrayBytes, fp);
}
void test_compress(FILE* outFp, FILE* inpFp)
{
Lizard_stream_t* lizardStream = Lizard_createStream_MinLevel();
char inpBuf[2][BLOCK_BYTES];
int inpBufIndex = 0;
if (!lizardStream) return;
lizardStream = Lizard_resetStream_MinLevel(lizardStream);
if (!lizardStream) return;
for(;;) {
char* const inpPtr = inpBuf[inpBufIndex];
const int inpBytes = (int) read_bin(inpFp, inpPtr, BLOCK_BYTES);
if(0 == inpBytes) {
break;
}
{
char cmpBuf[LIZARD_COMPRESSBOUND(BLOCK_BYTES)];
const int cmpBytes = Lizard_compress_continue(lizardStream, inpPtr, cmpBuf, inpBytes, sizeof(cmpBuf));
if(cmpBytes <= 0) {
break;
}
write_int(outFp, cmpBytes);
write_bin(outFp, cmpBuf, (size_t) cmpBytes);
}
inpBufIndex = (inpBufIndex + 1) % 2;
}
write_int(outFp, 0);
Lizard_freeStream(lizardStream);
}
void test_decompress(FILE* outFp, FILE* inpFp)
{
Lizard_streamDecode_t lizardStreamDecode_body;
Lizard_streamDecode_t* lizardStreamDecode = &lizardStreamDecode_body;
char decBuf[2][BLOCK_BYTES];
int decBufIndex = 0;
Lizard_setStreamDecode(lizardStreamDecode, NULL, 0);
for(;;) {
char cmpBuf[LIZARD_COMPRESSBOUND(BLOCK_BYTES)];
int cmpBytes = 0;
{
const size_t readCount0 = read_int(inpFp, &cmpBytes);
if(readCount0 != 1 || cmpBytes <= 0) {
break;
}
const size_t readCount1 = read_bin(inpFp, cmpBuf, (size_t) cmpBytes);
if(readCount1 != (size_t) cmpBytes) {
break;
}
}
{
char* const decPtr = decBuf[decBufIndex];
const int decBytes = Lizard_decompress_safe_continue(
lizardStreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES);
if(decBytes <= 0) {
break;
}
write_bin(outFp, decPtr, (size_t) decBytes);
}
decBufIndex = (decBufIndex + 1) % 2;
}
}
int compare(FILE* fp0, FILE* fp1)
{
int result = 0;
while(0 == result) {
char b0[65536];
char b1[65536];
const size_t r0 = read_bin(fp0, b0, sizeof(b0));
const size_t r1 = read_bin(fp1, b1, sizeof(b1));
result = (int) r0 - (int) r1;
if(0 == r0 || 0 == r1) {
break;
}
if(0 == result) {
result = memcmp(b0, b1, r0);
}
}
return result;
}
int main(int argc, char* argv[])
{
char inpFilename[256] = { 0 };
char lizardFilename[256] = { 0 };
char decFilename[256] = { 0 };
if(argc < 2) {
printf("Please specify input filename\n");
return 0;
}
snprintf(inpFilename, 256, "%s", argv[1]);
snprintf(lizardFilename, 256, "%s.lizs-%d", argv[1], BLOCK_BYTES);
snprintf(decFilename, 256, "%s.lizs-%d.dec", argv[1], BLOCK_BYTES);
printf("inp = [%s]\n", inpFilename);
printf("lizard = [%s]\n", lizardFilename);
printf("dec = [%s]\n", decFilename);
// compress
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* outFp = fopen(lizardFilename, "wb");
printf("compress : %s -> %s\n", inpFilename, lizardFilename);
test_compress(outFp, inpFp);
printf("compress : done\n");
fclose(outFp);
fclose(inpFp);
}
// decompress
{
FILE* inpFp = fopen(lizardFilename, "rb");
FILE* outFp = fopen(decFilename, "wb");
printf("decompress : %s -> %s\n", lizardFilename, decFilename);
test_decompress(outFp, inpFp);
printf("decompress : done\n");
fclose(outFp);
fclose(inpFp);
}
// verify
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* decFp = fopen(decFilename, "rb");
printf("verify : %s <-> %s\n", inpFilename, decFilename);
const int cmp = compare(inpFp, decFp);
if(0 == cmp) {
printf("verify : OK\n");
} else {
printf("verify : NG\n");
}
fclose(decFp);
fclose(inpFp);
}
return 0;
}

100
contrib/lizard/examples/blockStreaming_doubleBuffer.md Normal file
View File

@ -0,0 +1,100 @@
# Lizard streaming API Example : Double Buffer
by *Takayuki Matsuoka*
`blockStreaming_doubleBuffer.c` is Lizard Straming API example which implements double buffer (de)compression.
Please note :
- Firstly, read "Lizard Streaming API Basics".
- This is relatively advanced application example.
- Output file is not compatible with lizard_frame and platform dependent.
## What's the point of this example ?
- Handle huge file in small amount of memory
- Always better compression ratio than Block API
- Uniform block size
## How the compression works
First of all, allocate "Double Buffer" for input and Lizard compressed data buffer for output.
Double buffer has two pages, "first" page (Page#1) and "second" page (Page#2).
```
Double Buffer
Page#1 Page#2
+---------+---------+
| Block#1 | |
+----+----+---------+
|
v
{Out#1}
Prefix Dependency
+---------+
| |
v |
+---------+----+----+
| Block#1 | Block#2 |
+---------+----+----+
|
v
{Out#2}
External Dictionary Mode
+---------+
| |
| v
+----+----+---------+
| Block#3 | Block#2 |
+----+----+---------+
|
v
{Out#3}
Prefix Dependency
+---------+
| |
v |
+---------+----+----+
| Block#3 | Block#4 |
+---------+----+----+
|
v
{Out#4}
```
Next, read first block to double buffer's first page. And compress it by `Lizard_compress_continue()`.
For the first time, Lizard doesn't know any previous dependencies,
so it just compress the line without dependencies and generates compressed block {Out#1} to Lizard compressed data buffer.
After that, write {Out#1} to the file.
Next, read second block to double buffer's second page. And compress it.
In this time, Lizard can use dependency to Block#1 to improve compression ratio.
This dependency is called "Prefix mode".
Next, read third block to double buffer's *first* page. And compress it.
Also this time, Lizard can use dependency to Block#2.
This dependency is called "External Dictonaly mode".
Continue these procedure to the end of the file.
## How the decompression works
Decompression will do reverse order.
- Read first compressed block.
- Decompress it to the first page and write that page to the file.
- Read second compressed block.
- Decompress it to the second page and write that page to the file.
- Read third compressed block.
- Decompress it to the *first* page and write that page to the file.
Continue these procedure to the end of the compressed file.

210
contrib/lizard/examples/blockStreaming_lineByLine.c Normal file
View File

@ -0,0 +1,210 @@
// Lizard streaming API example : line-by-line logfile compression
// Copyright : Takayuki Matsuoka
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS
# define snprintf sprintf_s
#endif
#include "lizard_common.h"
#include "lizard_decompress.h"
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
static size_t write_uint16(FILE* fp, uint16_t i)
{
return fwrite(&i, sizeof(i), 1, fp);
}
static size_t write_bin(FILE* fp, const void* array, int arrayBytes)
{
return fwrite(array, 1, arrayBytes, fp);
}
static size_t read_uint16(FILE* fp, uint16_t* i)
{
return fread(i, sizeof(*i), 1, fp);
}
static size_t read_bin(FILE* fp, void* array, int arrayBytes)
{
return fread(array, 1, arrayBytes, fp);
}
static void test_compress(
FILE* outFp,
FILE* inpFp,
size_t messageMaxBytes,
size_t ringBufferBytes)
{
Lizard_stream_t* const lizardStream = Lizard_createStream_MinLevel();
const size_t cmpBufBytes = LIZARD_COMPRESSBOUND(messageMaxBytes);
char* const cmpBuf = (char*) malloc(cmpBufBytes);
char* const inpBuf = (char*) malloc(ringBufferBytes);
int inpOffset = 0;
for ( ; ; )
{
char* const inpPtr = &inpBuf[inpOffset];
#if 0
// Read random length data to the ring buffer.
const int randomLength = (rand() % messageMaxBytes) + 1;
const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
if (0 == inpBytes) break;
#else
// Read line to the ring buffer.
int inpBytes = 0;
if (!fgets(inpPtr, (int) messageMaxBytes, inpFp))
break;
inpBytes = (int) strlen(inpPtr);
#endif
{
const int cmpBytes = Lizard_compress_continue(lizardStream, inpPtr, cmpBuf, inpBytes, cmpBufBytes);
if (cmpBytes <= 0) break;
write_uint16(outFp, (uint16_t) cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
// Add and wraparound the ringbuffer offset
inpOffset += inpBytes;
if ((size_t)inpOffset >= ringBufferBytes - messageMaxBytes) inpOffset = 0;
}
}
write_uint16(outFp, 0);
free(inpBuf);
free(cmpBuf);
Lizard_freeStream(lizardStream);
}
static void test_decompress(
FILE* outFp,
FILE* inpFp,
size_t messageMaxBytes,
size_t ringBufferBytes)
{
Lizard_streamDecode_t* const lizardStreamDecode = Lizard_createStreamDecode();
char* const cmpBuf = (char*) malloc(LIZARD_COMPRESSBOUND(messageMaxBytes));
char* const decBuf = (char*) malloc(ringBufferBytes);
int decOffset = 0;
for ( ; ; )
{
uint16_t cmpBytes = 0;
if (read_uint16(inpFp, &cmpBytes) != 1) break;
if (cmpBytes <= 0) break;
if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break;
{
char* const decPtr = &decBuf[decOffset];
const int decBytes = Lizard_decompress_safe_continue(
lizardStreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes);
if (decBytes <= 0) break;
write_bin(outFp, decPtr, decBytes);
// Add and wraparound the ringbuffer offset
decOffset += decBytes;
if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0;
}
}
free(decBuf);
free(cmpBuf);
Lizard_freeStreamDecode(lizardStreamDecode);
}
static int compare(FILE* f0, FILE* f1)
{
int result = 0;
const size_t tempBufferBytes = 65536;
char* const b0 = (char*) malloc(tempBufferBytes);
char* const b1 = (char*) malloc(tempBufferBytes);
while(0 == result)
{
const size_t r0 = fread(b0, 1, tempBufferBytes, f0);
const size_t r1 = fread(b1, 1, tempBufferBytes, f1);
result = (int) r0 - (int) r1;
if (0 == r0 || 0 == r1) break;
if (0 == result) result = memcmp(b0, b1, r0);
}
free(b1);
free(b0);
return result;
}
int main(int argc, char* argv[])
{
enum {
MESSAGE_MAX_BYTES = 1024,
RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES,
};
char inpFilename[256] = { 0 };
char lizardFilename[256] = { 0 };
char decFilename[256] = { 0 };
if (argc < 2)
{
printf("Please specify input filename\n");
return 0;
}
snprintf(inpFilename, 256, "%s", argv[1]);
snprintf(lizardFilename, 256, "%s.lizs", argv[1]);
snprintf(decFilename, 256, "%s.lizs.dec", argv[1]);
printf("inp = [%s]\n", inpFilename);
printf("lizard = [%s]\n", lizardFilename);
printf("dec = [%s]\n", decFilename);
// compress
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* outFp = fopen(lizardFilename, "wb");
test_compress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES);
fclose(outFp);
fclose(inpFp);
}
// decompress
{
FILE* inpFp = fopen(lizardFilename, "rb");
FILE* outFp = fopen(decFilename, "wb");
test_decompress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES);
fclose(outFp);
fclose(inpFp);
}
// verify
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* decFp = fopen(decFilename, "rb");
const int cmp = compare(inpFp, decFp);
if (0 == cmp)
printf("Verify : OK\n");
else
printf("Verify : NG\n");
fclose(decFp);
fclose(inpFp);
}
return 0;
}

122
contrib/lizard/examples/blockStreaming_lineByLine.md Normal file
View File

@ -0,0 +1,122 @@
# Lizard streaming API Example : Line by Line Text Compression
by *Takayuki Matsuoka*
`blockStreaming_lineByLine.c` is Lizard Straming API example which implements line by line incremental (de)compression.
Please note the following restrictions :
- Firstly, read "Lizard Streaming API Basics".
- This is relatively advanced application example.
- Output file is not compatible with lizard_frame and platform dependent.
## What's the point of this example ?
- Line by line incremental (de)compression.
- Handle huge file in small amount of memory
- Generally better compression ratio than Block API
- Non-uniform block size
## How the compression works
First of all, allocate "Ring Buffer" for input and Lizard compressed data buffer for output.
```
(1)
Ring Buffer
+--------+
| Line#1 |
+---+----+
|
v
{Out#1}
(2)
Prefix Mode Dependency
+----+
| |
v |
+--------+-+------+
| Line#1 | Line#2 |
+--------+---+----+
|
v
{Out#2}
(3)
Prefix Prefix
+----+ +----+
| | | |
v | v |
+--------+-+------+-+------+
| Line#1 | Line#2 | Line#3 |
+--------+--------+---+----+
|
v
{Out#3}
(4)
External Dictionary Mode
+----+ +----+
| | | |
v | v |
------+--------+-+------+-+--------+
| .... | Line#X | Line#X+1 |
------+--------+--------+-----+----+
^ |
| v
| {Out#X+1}
|
Reset
(5)
Prefix
+-----+
| |
v |
------+--------+--------+----------+--+-------+
| .... | Line#X | Line#X+1 | Line#X+2 |
------+--------+--------+----------+-----+----+
^ |
| v
| {Out#X+2}
|
Reset
```
Next (see (1)), read first line to ringbuffer and compress it by `Lizard_compress_continue()`.
For the first time, Lizard doesn't know any previous dependencies,
so it just compress the line without dependencies and generates compressed line {Out#1} to Lizard compressed data buffer.
After that, write {Out#1} to the file and forward ringbuffer offset.
Do the same things to second line (see (2)).
But in this time, Lizard can use dependency to Line#1 to improve compression ratio.
This dependency is called "Prefix mode".
Eventually, we'll reach end of ringbuffer at Line#X (see (4)).
This time, we should reset ringbuffer offset.
After resetting, at Line#X+1 pointer is not adjacent, but Lizard still maintain its memory.
This is called "External Dictionary Mode".
In Line#X+2 (see (5)), finally Lizard forget almost all memories but still remains Line#X+1.
This is the same situation as Line#2.
Continue these procedure to the end of text file.
## How the decompression works
Decompression will do reverse order.
- Read compressed line from the file to buffer.
- Decompress it to the ringbuffer.
- Output decompressed plain text line to the file.
- Forward ringbuffer offset. If offset exceedes end of the ringbuffer, reset it.
Continue these procedure to the end of the compressed file.

202
contrib/lizard/examples/blockStreaming_ringBuffer.c Normal file
View File

@ -0,0 +1,202 @@
// Lizard streaming API example : ring buffer
// Based on sample code from Takayuki Matsuoka
/**************************************
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS // for MSVC
# define snprintf sprintf_s
#endif
#ifdef __GNUC__
# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
#endif
/**************************************
* Includes
**************************************/
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "lizard_common.h"
#include "lizard_decompress.h"
enum {
MESSAGE_MAX_BYTES = 1024,
RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES,
DECODE_RING_BUFFER = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES // Intentionally larger, to test unsynchronized ring buffers
};
size_t write_int32(FILE* fp, int32_t i) {
return fwrite(&i, sizeof(i), 1, fp);
}
size_t write_bin(FILE* fp, const void* array, int arrayBytes) {
return fwrite(array, 1, arrayBytes, fp);
}
size_t read_int32(FILE* fp, int32_t* i) {
return fread(i, sizeof(*i), 1, fp);
}
size_t read_bin(FILE* fp, void* array, int arrayBytes) {
return fread(array, 1, arrayBytes, fp);
}
void test_compress(FILE* outFp, FILE* inpFp)
{
Lizard_stream_t* lizardStream = Lizard_createStream_MinLevel();
if (!lizardStream) return;
static char inpBuf[RING_BUFFER_BYTES];
int inpOffset = 0;
for(;;) {
// Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer.
char* const inpPtr = &inpBuf[inpOffset];
const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1;
const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
if (0 == inpBytes) break;
{
char cmpBuf[LIZARD_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
const int cmpBytes = Lizard_compress_continue(lizardStream, inpPtr, cmpBuf, inpBytes, Lizard_compressBound(inpBytes));
if(cmpBytes <= 0) break;
write_int32(outFp, cmpBytes);
write_bin(outFp, cmpBuf, cmpBytes);
inpOffset += inpBytes;
// Wraparound the ringbuffer offset
if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) inpOffset = 0;
}
}
write_int32(outFp, 0);
Lizard_freeStream(lizardStream);
}
void test_decompress(FILE* outFp, FILE* inpFp)
{
static char decBuf[DECODE_RING_BUFFER];
int decOffset = 0;
Lizard_streamDecode_t lizardStreamDecode_body = { 0 };
Lizard_streamDecode_t* lizardStreamDecode = &lizardStreamDecode_body;
for(;;) {
int cmpBytes = 0;
char cmpBuf[LIZARD_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
{
const size_t r0 = read_int32(inpFp, &cmpBytes);
if(r0 != 1 || cmpBytes <= 0) break;
const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes);
if(r1 != (size_t) cmpBytes) break;
}
{
char* const decPtr = &decBuf[decOffset];
const int decBytes = Lizard_decompress_safe_continue(
lizardStreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
if(decBytes <= 0) break;
decOffset += decBytes;
write_bin(outFp, decPtr, decBytes);
// Wraparound the ringbuffer offset
if(decOffset >= DECODE_RING_BUFFER - MESSAGE_MAX_BYTES) decOffset = 0;
}
}
}
int compare(FILE* f0, FILE* f1)
{
int result = 0;
while(0 == result) {
char b0[65536];
char b1[65536];
const size_t r0 = fread(b0, 1, sizeof(b0), f0);
const size_t r1 = fread(b1, 1, sizeof(b1), f1);
result = (int) r0 - (int) r1;
if(0 == r0 || 0 == r1) {
break;
}
if(0 == result) {
result = memcmp(b0, b1, r0);
}
}
return result;
}
int main(int argc, char** argv)
{
char inpFilename[256] = { 0 };
char lizardFilename[256] = { 0 };
char decFilename[256] = { 0 };
if(argc < 2) {
printf("Please specify input filename\n");
return 0;
}
snprintf(inpFilename, 256, "%s", argv[1]);
snprintf(lizardFilename, 256, "%s.lizs-%d", argv[1], 0);
snprintf(decFilename, 256, "%s.lizs-%d.dec", argv[1], 0);
printf("inp = [%s]\n", inpFilename);
printf("lizard = [%s]\n", lizardFilename);
printf("dec = [%s]\n", decFilename);
// compress
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* outFp = fopen(lizardFilename, "wb");
test_compress(outFp, inpFp);
fclose(outFp);
fclose(inpFp);
}
// decompress
{
FILE* inpFp = fopen(lizardFilename, "rb");
FILE* outFp = fopen(decFilename, "wb");
test_decompress(outFp, inpFp);
fclose(outFp);
fclose(inpFp);
}
// verify
{
FILE* inpFp = fopen(inpFilename, "rb");
FILE* decFp = fopen(decFilename, "rb");
const int cmp = compare(inpFp, decFp);
if(0 == cmp) {
printf("Verify : OK\n");
} else {
printf("Verify : NG\n");
}
fclose(decFp);
fclose(inpFp);
}
return 0;
}

303
contrib/lizard/examples/compress_functions.c Normal file
View File

@ -0,0 +1,303 @@
/*
* compress_functions.c
* Copyright : Kyle Harper
* License : Follows same licensing as the lizard_compress.c/lizard_compress.h program at any given time. Currently, BSD 2.
* Description: A program to demonstrate the various compression functions involved in when using Lizard_compress_MinLevel(). The idea
* is to show how each step in the call stack can be used directly, if desired. There is also some benchmarking for
* each function to demonstrate the (probably lack of) performance difference when jumping the stack.
* (If you're new to lizard, please read simple_buffer.c to understand the fundamentals)
*
* The call stack (before theoretical compiler optimizations) for Lizard_compress_MinLevel is as follows:
* Lizard_compress_MinLevel
* Lizard_compress_fast
* Lizard_compress_extState_MinLevel
* Lizard_compress_generic
*
* Lizard_compress_MinLevel()
* This is the recommended function for compressing data. It will serve as the baseline for comparison.
* Lizard_compress_fast()
* Despite its name, it's not a "fast" version of compression. It simply decides if HEAPMODE is set and either
* allocates memory on the heap for a struct or creates the struct directly on the stack. Stack access is generally
* faster but this function itself isn't giving that advantage, it's just some logic for compile time.
* Lizard_compress_extState_MinLevel()
* This simply accepts all the pointers and values collected thus far and adds logic to determine how
* Lizard_compress_generic should be invoked; specifically: can the source fit into a single pass as determined by
* Lizard_64Klimit.
* Lizard_compress_generic()
* As the name suggests, this is the generic function that ultimately does most of the heavy lifting. Calling this
* directly can help avoid some test cases and branching which might be useful in some implementation-specific
* situations, but you really need to know what you're doing AND what you're asking lizard to do! You also need a
* wrapper function because this function isn't exposed with lizard_compress.h.
*
* The call stack for decompression functions is shallow. There are 2 options:
* Lizard_decompress_safe || Lizard_decompress_fast
* Lizard_decompress_generic
*
* Lizard_decompress_safe
* This is the recommended function for decompressing data. It is considered safe because the caller specifies
* both the size of the compresssed buffer to read as well as the maximum size of the output (decompressed) buffer
* instead of just the latter.
* Lizard_decompress_generic
* This is the generic function that both of the Lizard_decompress_* functions above end up calling. Calling this
* directly is not advised, period. Furthermore, it is a static inline function in lizard_compress.c, so there isn't a symbol
* exposed for anyone using lizard_compress.h to utilize.
*
* Special Note About Decompression:
* Using the Lizard_decompress_safe() function protects against malicious (user) input.
*/
/* Since lizard compiles with c99 and not gnu/std99 we need to enable POSIX linking for time.h structs and functions. */
#if __STDC_VERSION__ >= 199901L
#define _XOPEN_SOURCE 600
#else
#define _XOPEN_SOURCE 500
#endif
#define _POSIX_C_SOURCE 199309L
/* Includes, for Power! */
#include "lizard_compress.h"
#include "lizard_decompress.h"
#include <stdio.h> /* for printf() */
#include <stdlib.h> /* for exit() */
#include <string.h> /* for atoi() memcmp() */
#include <stdint.h> /* for uint_types */
#include <inttypes.h> /* for PRIu64 */
#include <time.h> /* for clock_gettime() */
#include <locale.h> /* for setlocale() */
/* We need to know what one billion is for clock timing. */
#define BILLION 1000000000L
/* Create a crude set of test IDs so we can switch on them later (Can't switch() on a char[] or char*). */
#define ID__LIZARD_COMPRESS_DEFAULT 1
#define ID__LIZARD_COMPRESS_GENERIC 4
#define ID__LIZARD_DECOMPRESS_SAFE 5
/*
* Easy show-error-and-bail function.
*/
void run_screaming(const char *message, const int code) {
printf("%s\n", message);
exit(code);
return;
}
/*
* Centralize the usage function to keep main cleaner.
*/
void usage(const char *message) {
printf("Usage: ./argPerformanceTesting <iterations>\n");
run_screaming(message, 1);
return;
}
/*
* Runs the benchmark for Lizard_compress_* based on function_id.
*/
uint64_t bench(
const char *known_good_dst,
const int function_id,
const int iterations,
const char *src,
char *dst,
const size_t src_size,
const size_t max_dst_size,
const size_t comp_size
) {
uint64_t time_taken = 0;
int rv = 0;
const int warm_up = 5000;
struct timespec start, end;
Lizard_stream_t* state = Lizard_createStream_MinLevel();
if (!state) return;
// Select the right function to perform the benchmark on. We perform 5000 initial loops to warm the cache and ensure that dst
// remains matching to known_good_dst between successive calls.
switch(function_id) {
case ID__LIZARD_COMPRESS_DEFAULT:
printf("Starting benchmark for function: Lizard_compress_MinLevel()\n");
for(int junk=0; junk<warm_up; junk++)
rv = Lizard_compress_MinLevel(src, dst, src_size, max_dst_size);
if (rv < 1)
run_screaming("Couldn't run Lizard_compress_MinLevel()... error code received is in exit code.", rv);
if (memcmp(known_good_dst, dst, max_dst_size) != 0)
run_screaming("According to memcmp(), the compressed dst we got doesn't match the known_good_dst... ruh roh.", 1);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i=1; i<=iterations; i++)
Lizard_compress_MinLevel(src, dst, src_size, max_dst_size);
break;
// Disabled until Lizard_compress_generic() is exposed in the header.
// case ID__LIZARD_COMPRESS_GENERIC:
// printf("Starting benchmark for function: Lizard_compress_generic()\n");
// Lizard_resetStream_MinLevel((Lizard_stream_t*)state);
// for(int junk=0; junk<warm_up; junk++) {
// Lizard_resetStream_MinLevel((Lizard_stream_t*)state);
// //rv = Lizard_compress_generic_wrapper(state, src, dst, src_size, max_dst_size, notLimited, byU16, noDict, noDictIssue);
// Lizard_compress_generic_wrapper(state, src, dst, src_size, max_dst_size);
// }
// if (rv < 1)
// run_screaming("Couldn't run Lizard_compress_generic()... error code received is in exit code.", rv);
// if (memcmp(known_good_dst, dst, max_dst_size) != 0)
// run_screaming("According to memcmp(), the compressed dst we got doesn't match the known_good_dst... ruh roh.", 1);
// for (int i=1; i<=iterations; i++) {
// Lizard_resetStream_MinLevel((Lizard_stream_t*)state);
// //Lizard_compress_generic_wrapper(state, src, dst, src_size, max_dst_size, notLimited, byU16, noDict, noDictIssue, acceleration);
// Lizard_compress_generic_wrapper(state, src, dst, src_size, max_dst_size, acceleration);
// }
// break;
case ID__LIZARD_DECOMPRESS_SAFE:
printf("Starting benchmark for function: Lizard_decompress_safe()\n");
for(int junk=0; junk<warm_up; junk++)
rv = Lizard_decompress_safe(src, dst, comp_size, src_size);
if (rv < 1)
run_screaming("Couldn't run Lizard_decompress_safe()... error code received is in exit code.", rv);
if (memcmp(known_good_dst, dst, src_size) != 0)
run_screaming("According to memcmp(), the compressed dst we got doesn't match the known_good_dst... ruh roh.", 1);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i=1; i<=iterations; i++)
Lizard_decompress_safe(src, dst, comp_size, src_size);
break;
default:
run_screaming("The test specified isn't valid. Please check your code.", 1);
break;
}
// Stop timer and return time taken.
clock_gettime(CLOCK_MONOTONIC, &end);
time_taken = BILLION *(end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec;
Lizard_freeStream(state);
return time_taken;
}
/*
* main()
* We will demonstrate the use of each function for simplicity sake. Then we will run 2 suites of benchmarking:
* Test suite A) Uses generic Lorem Ipsum text which should be generally compressible insomuch as basic human text is
* compressible for such a small src_size
* Test Suite B) For the sake of testing, see what results we get if the data is drastically easier to compress. IF there are
* indeed losses and IF more compressible data is faster to process, this will exacerbate the findings.
*/
int main(int argc, char **argv) {
// Get and verify options. There's really only 1: How many iterations to run.
int iterations = 1000000;
if (argc > 1)
iterations = atoi(argv[1]);
if (iterations < 1)
usage("Argument 1 (iterations) must be > 0.");
// First we will create 2 sources (char *) of 2000 bytes each. One normal text, the other highly-compressible text.
const char *src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed luctus purus et risus vulputate, et mollis orci ullamcorper. Nulla facilisi. Fusce in ligula sed purus varius aliquet interdum vitae justo. Proin quis diam velit. Nulla varius iaculis auctor. Cras volutpat, justo eu dictum pulvinar, elit sem porttitor metus, et imperdiet metus sapien et ante. Nullam nisi nulla, ornare eu tristique eu, dignissim vitae diam. Nulla sagittis porta libero, a accumsan felis sagittis scelerisque. Integer laoreet eleifend congue. Etiam rhoncus leo vel dolor fermentum, quis luctus nisl iaculis. Praesent a erat sapien. Aliquam semper mi in lorem ultrices ultricies. Lorem ipsum dolor sit amet, consectetur adipiscing elit. In feugiat risus sed enim ultrices, at sodales nulla tristique. Maecenas eget pellentesque justo, sed pellentesque lectus. Fusce sagittis sit amet elit vel varius. Donec sed ligula nec ligula vulputate rutrum sed ut lectus. Etiam congue pharetra leo vitae cursus. Morbi enim ante, porttitor ut varius vel, tincidunt quis justo. Nunc iaculis, risus id ultrices semper, metus est efficitur ligula, vel posuere risus nunc eget purus. Ut lorem turpis, condimentum at sem sed, porta aliquam turpis. In ut sapien a nulla dictum tincidunt quis sit amet lorem. Fusce at est egestas, luctus neque eu, consectetur tortor. Phasellus eleifend ultricies nulla ac lobortis. Morbi maximus quam cursus vehicula iaculis. Maecenas cursus vel justo ut rutrum. Curabitur magna orci, dignissim eget dapibus vitae, finibus id lacus. Praesent rhoncus mattis augue vitae bibendum. Praesent porta mauris non ultrices fermentum. Quisque vulputate ipsum in sodales pulvinar. Aliquam nec mollis felis. Donec vitae augue pulvinar, congue nisl sed, pretium purus. Fusce lobortis mi ac neque scelerisque semper. Pellentesque vel est vitae magna aliquet aliquet. Nam non dolor. Nulla facilisi. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Morbi ac lacinia felis metus.";
const char *hc_src = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
// Set and derive sizes. Since we're using strings, use strlen() + 1 for \0.
const size_t src_size = strlen(src) + 1;
const size_t max_dst_size = Lizard_compressBound(src_size);
int bytes_returned = 0;
// Now build allocations for the data we'll be playing with.
char *dst = calloc(1, max_dst_size);
char *known_good_dst = calloc(1, max_dst_size);
char *known_good_hc_dst = calloc(1, max_dst_size);
if (dst == NULL || known_good_dst == NULL || known_good_hc_dst == NULL)
run_screaming("Couldn't allocate memory for the destination buffers. Sad :(", 1);
// Create known-good buffers to verify our tests with other functions will produce the same results.
bytes_returned = Lizard_compress_MinLevel(src, known_good_dst, src_size, max_dst_size);
if (bytes_returned < 1)
run_screaming("Couldn't create a known-good destination buffer for comparison... this is bad.", 1);
const size_t src_comp_size = bytes_returned;
bytes_returned = Lizard_compress_MinLevel(hc_src, known_good_hc_dst, src_size, max_dst_size);
if (bytes_returned < 1)
run_screaming("Couldn't create a known-good (highly compressible) destination buffer for comparison... this is bad.", 1);
const size_t hc_src_comp_size = bytes_returned;
/* Lizard_compress_MinLevel() */
// This is the default function so we don't need to demonstrate how to use it. See basics.c if you need more basal information.
/* Lizard_compress_extState_MinLevel() */
// Using this function directly requires that we build an Lizard_stream_t struct ourselves. We do NOT have to reset it ourselves.
memset(dst, 0, max_dst_size);
Lizard_stream_t* state = Lizard_createStream_MinLevel();
if (!state) return;
bytes_returned = Lizard_compress_extState_MinLevel(state, src, dst, src_size, max_dst_size, 1);
if (bytes_returned < 1)
run_screaming("Failed to compress src using Lizard_compress_extState_MinLevel. echo $? for return code.", bytes_returned);
if (memcmp(dst, known_good_dst, bytes_returned) != 0)
run_screaming("According to memcmp(), the value we got in dst from Lizard_compress_extState_MinLevel doesn't match the known-good value. This is bad.", 1);
/* Lizard_compress_generic */
// When you can exactly control the inputs and options of your Lizard needs, you can use Lizard_compress_generic and fixed (const)
// values for the enum types such as dictionary and limitations. Any other direct-use is probably a bad idea.
//
// That said, the Lizard_compress_generic() function is 'static inline' and does not have a prototype in lizard_compress.h to expose a symbol
// for it. In other words: we can't access it directly. I don't want to submit a PR that modifies lizard_compress.c/h. Yann and others can
// do that if they feel it's worth expanding this example.
//
// I will, however, leave a skeleton of what would be required to use it directly:
/*
memset(dst, 0, max_dst_size);
// Lizard_stream_t state: is already declared above. We can reuse it BUT we have to reset the stream ourselves between each call.
Lizard_resetStream_MinLevel((Lizard_stream_t *)state);
// Since src size is small we know the following enums will be used: notLimited (0), byU16 (2), noDict (0), noDictIssue (0).
bytes_returned = Lizard_compress_generic(state, src, dst, src_size, max_dst_size, notLimited, byU16, noDict, noDictIssue, 1);
if (bytes_returned < 1)
run_screaming("Failed to compress src using Lizard_compress_generic. echo $? for return code.", bytes_returned);
if (memcmp(dst, known_good_dst, bytes_returned) != 0)
run_screaming("According to memcmp(), the value we got in dst from Lizard_compress_generic doesn't match the known-good value. This is bad.", 1);
*/
Lizard_freeStream(state);
/* Benchmarking */
/* Now we'll run a few rudimentary benchmarks with each function to demonstrate differences in speed based on the function used.
* Remember, we cannot call Lizard_compress_generic() directly (yet) so it's disabled.
*/
// Suite A - Normal Compressibility
char *dst_d = calloc(1, src_size);
memset(dst, 0, max_dst_size);
printf("\nStarting suite A: Normal compressible text.\n");
uint64_t time_taken__default = bench(known_good_dst, ID__LIZARD_COMPRESS_DEFAULT, iterations, src, dst, src_size, max_dst_size, src_comp_size);
//uint64_t time_taken__generic = bench(known_good_dst, ID__LIZARD_COMPRESS_GENERIC, iterations, src, dst, src_size, max_dst_size, src_comp_size);
uint64_t time_taken__decomp_safe = bench(src, ID__LIZARD_DECOMPRESS_SAFE, iterations, known_good_dst, dst_d, src_size, max_dst_size, src_comp_size);
// Suite B - Highly Compressible
memset(dst, 0, max_dst_size);
printf("\nStarting suite B: Highly compressible text.\n");
uint64_t time_taken_hc__default = bench(known_good_hc_dst, ID__LIZARD_COMPRESS_DEFAULT, iterations, hc_src, dst, src_size, max_dst_size, hc_src_comp_size);
//uint64_t time_taken_hc__generic = bench(known_good_hc_dst, ID__LIZARD_COMPRESS_GENERIC, iterations, hc_src, dst, src_size, max_dst_size, hc_src_comp_size);
uint64_t time_taken_hc__decomp_safe = bench(hc_src, ID__LIZARD_DECOMPRESS_SAFE, iterations, known_good_hc_dst, dst_d, src_size, max_dst_size, hc_src_comp_size);
// Report and leave.
setlocale(LC_ALL, "");
const char *format = "|%-14s|%-30s|%'14.9f|%'16d|%'14d|%'13.2f%%|\n";
const char *header_format = "|%-14s|%-30s|%14s|%16s|%14s|%14s|\n";
const char *separator = "+--------------+------------------------------+--------------+----------------+--------------+--------------+\n";
printf("\n");
printf("%s", separator);
printf(header_format, "Source", "Function Benchmarked", "Total Seconds", "Iterations/sec", "ns/Iteration", "% of default");
printf("%s", separator);
printf(format, "Normal Text", "Lizard_compress_MinLevel()", (double)time_taken__default / BILLION, (int)(iterations / ((double)time_taken__default /BILLION)), time_taken__default / iterations, (double)time_taken__default * 100 / time_taken__default);
printf(format, "Normal Text", "Lizard_compress_fast()", (double)time_taken__fast / BILLION, (int)(iterations / ((double)time_taken__fast /BILLION)), time_taken__fast / iterations, (double)time_taken__fast * 100 / time_taken__default);
printf(format, "Normal Text", "Lizard_compress_extState_MinLevel()", (double)time_taken__fast_extstate / BILLION, (int)(iterations / ((double)time_taken__fast_extstate /BILLION)), time_taken__fast_extstate / iterations, (double)time_taken__fast_extstate * 100 / time_taken__default);
//printf(format, "Normal Text", "Lizard_compress_generic()", (double)time_taken__generic / BILLION, (int)(iterations / ((double)time_taken__generic /BILLION)), time_taken__generic / iterations, (double)time_taken__generic * 100 / time_taken__default);
printf(format, "Normal Text", "Lizard_decompress_safe()", (double)time_taken__decomp_safe / BILLION, (int)(iterations / ((double)time_taken__decomp_safe /BILLION)), time_taken__decomp_safe / iterations, (double)time_taken__decomp_safe * 100 / time_taken__default);
printf(header_format, "", "", "", "", "", "");
printf(format, "Compressible", "Lizard_compress_MinLevel()", (double)time_taken_hc__default / BILLION, (int)(iterations / ((double)time_taken_hc__default /BILLION)), time_taken_hc__default / iterations, (double)time_taken_hc__default * 100 / time_taken_hc__default);
printf(format, "Compressible", "Lizard_compress_fast()", (double)time_taken_hc__fast / BILLION, (int)(iterations / ((double)time_taken_hc__fast /BILLION)), time_taken_hc__fast / iterations, (double)time_taken_hc__fast * 100 / time_taken_hc__default);
printf(format, "Compressible", "Lizard_compress_extState_MinLevel()", (double)time_taken_hc__fast_extstate / BILLION, (int)(iterations / ((double)time_taken_hc__fast_extstate /BILLION)), time_taken_hc__fast_extstate / iterations, (double)time_taken_hc__fast_extstate * 100 / time_taken_hc__default);
//printf(format, "Compressible", "Lizard_compress_generic()", (double)time_taken_hc__generic / BILLION, (int)(iterations / ((double)time_taken_hc__generic /BILLION)), time_taken_hc__generic / iterations, (double)time_taken_hc__generic * 100 / time_taken_hc__default);
printf(format, "Compressible", "Lizard_decompress_safe()", (double)time_taken_hc__decomp_safe / BILLION, (int)(iterations / ((double)time_taken_hc__decomp_safe /BILLION)), time_taken_hc__decomp_safe / iterations, (double)time_taken_hc__decomp_safe * 100 / time_taken_hc__default);
printf("%s", separator);
printf("\n");
printf("All done, ran %d iterations per test.\n", iterations);
return 0;
}

169
contrib/lizard/examples/frameCompress.c Normal file
View File

@ -0,0 +1,169 @@
// Lizardframe API example : compress a file
// Based on sample code from Zbigniew Jędrzejewski-Szmek
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <lizard_frame.h>
#define BUF_SIZE (16*1024)
#define LIZARD_HEADER_SIZE 19
#define LIZARD_FOOTER_SIZE 4
static const LizardF_preferences_t lizard_preferences = {
{ LizardF_max256KB, LizardF_blockLinked, LizardF_noContentChecksum, LizardF_frame, 0, { 0, 0 } },
0, /* compression level */
0, /* autoflush */
{ 0, 0, 0, 0 }, /* reserved, must be set to 0 */
};
static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) {
LizardF_errorCode_t r;
LizardF_compressionContext_t ctx;
char *src, *buf = NULL;
size_t size, n, k, count_in = 0, count_out, offset = 0, frame_size;
r = LizardF_createCompressionContext(&ctx, LIZARDF_VERSION);
if (LizardF_isError(r)) {
printf("Failed to create context: error %zu", r);
return 1;
}
r = 1;
src = malloc(BUF_SIZE);
if (!src) {
printf("Not enough memory");
goto cleanup;
}
frame_size = LizardF_compressBound(BUF_SIZE, &lizard_preferences);
size = frame_size + LIZARD_HEADER_SIZE + LIZARD_FOOTER_SIZE;
buf = malloc(size);
if (!buf) {
printf("Not enough memory");
goto cleanup;
}
n = offset = count_out = LizardF_compressBegin(ctx, buf, size, &lizard_preferences);
if (LizardF_isError(n)) {
printf("Failed to start compression: error %zu", n);
goto cleanup;
}
printf("Buffer size is %zu bytes, header size %zu bytes\n", size, n);
for (;;) {
k = fread(src, 1, BUF_SIZE, in);
if (k == 0)
break;
count_in += k;
n = LizardF_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL);
if (LizardF_isError(n)) {
printf("Compression failed: error %zu", n);
goto cleanup;
}
offset += n;
count_out += n;
if (size - offset < frame_size + LIZARD_FOOTER_SIZE) {
printf("Writing %zu bytes\n", offset);
k = fwrite(buf, 1, offset, out);
if (k < offset) {
if (ferror(out))
printf("Write failed");
else
printf("Short write");
goto cleanup;
}
offset = 0;
}
}
n = LizardF_compressEnd(ctx, buf + offset, size - offset, NULL);
if (LizardF_isError(n)) {
printf("Failed to end compression: error %zu", n);
goto cleanup;
}
offset += n;
count_out += n;
printf("Writing %zu bytes\n", offset);
k = fwrite(buf, 1, offset, out);
if (k < offset) {
if (ferror(out))
printf("Write failed");
else
printf("Short write");
goto cleanup;
}
*size_in = count_in;
*size_out = count_out;
r = 0;
cleanup:
if (ctx)
LizardF_freeCompressionContext(ctx);
free(src);
free(buf);
return r;
}
static int compress(const char *input, const char *output) {
char *tmp = NULL;
FILE *in = NULL, *out = NULL;
size_t size_in = 0, size_out = 0;
int r = 1;
if (!output) {
size_t len = strlen(input);
output = tmp = malloc(len + 5);
if (!tmp) {
printf("Not enough memory");
return 1;
}
strcpy(tmp, input);
strcpy(tmp + len, ".liz");
}
in = fopen(input, "rb");
if (!in) {
fprintf(stderr, "Failed to open input file %s: %s\n", input, strerror(errno));
goto cleanup;
}
out = fopen(output, "wb");
if (!out) {
fprintf(stderr, "Failed to open output file %s: %s\n", output, strerror(errno));
goto cleanup;
}
r = compress_file(in, out, &size_in, &size_out);
if (r == 0)
printf("%s: %zu → %zu bytes, %.1f%%\n",
input, size_in, size_out,
(double)size_out / size_in * 100);
cleanup:
if (in)
fclose(in);
if (out)
fclose(out);
free(tmp);
return r;
}
int main(int argc, char **argv) {
if (argc < 2 || argc > 3) {
fprintf(stderr, "Syntax: %s <input> <output>\n", argv[0]);
return EXIT_FAILURE;
}
return compress(argv[1], argv[2]);
}

13
contrib/lizard/examples/printVersion.c Normal file
View File

@ -0,0 +1,13 @@
// Lizard trivial example : print Library version number
// Copyright : Takayuki Matsuoka & Yann Collet
#include <stdio.h>
#include "lizard_compress.h"
int main(int argc, char** argv)
{
(void)argc; (void)argv;
printf("Hello World ! Lizard Library version = %d\n", Lizard_versionNumber());
return 0;
}

91
contrib/lizard/examples/simple_buffer.c Normal file
View File

@ -0,0 +1,91 @@
/*
* simple_buffer.c
* Copyright : Kyle Harper
* License : Follows same licensing as the lizard_compress.c/lizard_compress.h program at any given time. Currently, BSD 2.
* Description: Example program to demonstrate the basic usage of the compress/decompress functions within lizard_compress.c/lizard_compress.h.
* The functions you'll likely want are Lizard_compress_MinLevel and Lizard_decompress_safe. Both of these are documented in
* the lizard_compress.h header file; I recommend reading them.
*/
/* Includes, for Power! */
#include "lizard_compress.h" // This is all that is required to expose the prototypes for basic compression and decompression.
#include "lizard_decompress.h"
#include <stdio.h> // For printf()
#include <string.h> // For memcmp()
#include <stdlib.h> // For exit()
/*
* Easy show-error-and-bail function.
*/
void run_screaming(const char *message, const int code) {
printf("%s\n", message);
exit(code);
return;
}
/*
* main
*/
int main(void) {
/* Introduction */
// Below we will have a Compression and Decompression section to demonstrate. There are a few important notes before we start:
// 1) The return codes of Lizard_ functions are important. Read lizard_compress.h if you're unsure what a given code means.
// 2) Lizard uses char* pointers in all Lizard_ functions. This is baked into the API and probably not going to change. If your
// program uses pointers that are unsigned char*, void*, or otherwise different you may need to do some casting or set the
// right -W compiler flags to ignore those warnings (e.g.: -Wno-pointer-sign).
/* Compression */
// We'll store some text into a variable pointed to by *src to be compressed later.
const char *src = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
// The compression function needs to know how many bytes of exist. Since we're using a string, we can use strlen() + 1 (for \0).
const size_t src_size = strlen(src) + 1;
// Lizard provides a function that will tell you the maximum size of compressed output based on input data via Lizard_compressBound().
const size_t max_dst_size = Lizard_compressBound(src_size);
// We will use that size for our destination boundary when allocating space.
char *compressed_data = malloc(max_dst_size);
if (compressed_data == NULL)
run_screaming("Failed to allocate memory for *compressed_data.", 1);
// That's all the information and preparation Lizard needs to compress *src into *compressed_data. Invoke Lizard_compress_MinLevel now
// with our size values and pointers to our memory locations. Save the return value for error checking.
int return_value = 0;
return_value = Lizard_compress_MinLevel(src, compressed_data, src_size, max_dst_size);
// Check return_value to determine what happened.
if (return_value < 0)
run_screaming("A negative result from Lizard_compress_MinLevel indicates a failure trying to compress the data. See exit code (echo $?) for value returned.", return_value);
if (return_value == 0)
run_screaming("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.", 1);
if (return_value > 0)
printf("We successfully compressed some data!\n");
// Not only does a positive return_value mean success, the value returned == the number of bytes required. You can use this to
// realloc() *compress_data to free up memory, if desired. We'll do so just to demonstrate the concept.
const size_t compressed_data_size = return_value;
compressed_data = (char *)realloc(compressed_data, compressed_data_size);
if (compressed_data == NULL)
run_screaming("Failed to re-alloc memory for compressed_data. Sad :(", 1);
/* Decompression */
// Now that we've successfully compressed the information from *src to *compressed_data, let's do the opposite! We'll create a
// *new_src location of size src_size since we know that value.
char *new_src = malloc(src_size);
if (new_src == NULL)
run_screaming("Failed to allocate memory for *new_src.", 1);
// The Lizard_decompress_safe function needs to know where the compressed data is, how many bytes long it is, where the new_src
// memory location is, and how large the new_src (uncompressed) output will be. Again, save the return_value.
return_value = Lizard_decompress_safe(compressed_data, new_src, compressed_data_size, src_size);
if (return_value < 0)
run_screaming("A negative result from Lizard_decompress_fast indicates a failure trying to decompress the data. See exit code (echo $?) for value returned.", return_value);
if (return_value == 0)
run_screaming("I'm not sure this function can ever return 0. Documentation in lizard_compress.h doesn't indicate so.", 1);
if (return_value > 0)
printf("We successfully decompressed some data!\n");
// Not only does a positive return value mean success, the value returned == the number of bytes read from the compressed_data
// stream. I'm not sure there's ever a time you'll need to know this in most cases...
/* Validation */
// We should be able to compare our original *src with our *new_src and be byte-for-byte identical.
if (memcmp(src, new_src, src_size) != 0)
run_screaming("Validation failed. *src and *new_src are not identical.", 1);
printf("Validation done. The string we ended up with is:\n%s\n", new_src);
return 0;
}

87
contrib/lizard/examples/streaming_api_basics.md Normal file
View File

@ -0,0 +1,87 @@
# Lizard Streaming API Basics
by *Takayuki Matsuoka*
## Lizard API sets
Lizard has the following API sets :
- "Auto Framing" API (lizard_frame.h) :
This is most recommended API for usual application.
It guarantees interoperability with other Lizard framing format compliant tools/libraries
such as Lizard command line utility, node-lizard, etc.
- "Block" API : This is recommended for simple purpose.
It compress single raw memory block to Lizard memory block and vice versa.
- "Streaming" API : This is designed for complex thing.
For example, compress huge stream data in restricted memory environment.
Basically, you should use "Auto Framing" API.
But if you want to write advanced application, it's time to use Block or Streaming APIs.
## What is difference between Block and Streaming API ?
Block API (de)compresses single contiguous memory block.
In other words, Lizard library find redundancy from single contiguous memory block.
Streaming API does same thing but (de)compress multiple adjacent contiguous memory block.
So Lizard library could find more redundancy than Block API.
The following figure shows difference between API and block sizes.
In these figures, original data is splitted to 4KiBytes contiguous chunks.
```
Original Data
+---------------+---------------+----+----+----+
| 4KiB Chunk A | 4KiB Chunk B | C | D |... |
+---------------+---------------+----+----+----+
Example (1) : Block API, 4KiB Block
+---------------+---------------+----+----+----+
| 4KiB Chunk A | 4KiB Chunk B | C | D |... |
+---------------+---------------+----+----+----+
| Block #1 | Block #2 | #3 | #4 |... |
+---------------+---------------+----+----+----+
(No Dependency)
Example (2) : Block API, 8KiB Block
+---------------+---------------+----+----+----+
| 4KiB Chunk A | 4KiB Chunk B | C | D |... |
+---------------+---------------+----+----+----+
| Block #1 |Block #2 |... |
+--------------------+----------+-------+-+----+
^ | ^ |
| | | |
+--------------+ +----+
Internal Dependency Internal Dependency
Example (3) : Streaming API, 4KiB Block
+---------------+---------------+-----+----+----+
| 4KiB Chunk A | 4KiB Chunk B | C | D |... |
+---------------+---------------+-----+----+----+
| Block #1 | Block #2 | #3 | #4 |... |
+---------------+----+----------+-+---+-+--+----+
^ | ^ | ^ |
| | | | | |
+--------------+ +--------+ +---+
Dependency Dependency Dependency
```
- In example (1), there is no dependency.
All blocks are compressed independently.
- In example (2), naturally 8KiBytes block has internal dependency.
But still block #1 and #2 are compressed independently.
- In example (3), block #2 has dependency to #1,
also #3 has dependency to #2 and #1, #4 has #3, #2 and #1, and so on.
Here, we can observe difference between example (2) and (3).
In (2), there's no dependency between chunk B and C, but (3) has dependency between B and C.
This dependency improves compression ratio.
## Restriction of Streaming API
For the efficiency, Streaming API doesn't keep mirror copy of dependent (de)compressed memory.
This means users should keep these dependent (de)compressed memory explicitly.
Usually, "Dependent memory" is previous adjacent contiguous memory up to 64KiBytes.
Lizard will not access further memories.

3
contrib/lizard/lib/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
# make install artefact
liblizard.pc
test/

25
contrib/lizard/lib/LICENSE Normal file
View File

@ -0,0 +1,25 @@
Lizard Library
Copyright (C) 2011-2016, Yann Collet.
Copyright (C) 2016-2017, Przemyslaw Skibinski <inikep@gmail.com>
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

76
contrib/lizard/lib/README.md Normal file
View File

@ -0,0 +1,76 @@
Lizard - Library Files
================================
The __lib__ directory contains several directories.
Depending on target use case, it's enough to include only files from relevant directories.
#### API
Lizard stable API is exposed within [lizard_compress.h](lizard_compress.h) and [lizard_decompress.h](lizard_decompress.h),
at the root of `lib` directory.
#### Compatibility issues
The raw Lizard block compression format is detailed within [lizard_Block_format].
To compress an arbitrarily long file or data stream, multiple blocks are required.
Organizing these blocks and providing a common header format to handle their content
is the purpose of the Frame format, defined in [lizard_Frame_format].
`lizard` command line utility produces files or streams compatible with the Frame format.
(_Advanced stuff_ : It's possible to hide xxhash symbols into a local namespace.
This is what `liblizard` does, to avoid symbol duplication
in case a user program would link to several libraries containing xxhash symbols.)
[lizard_Block_format]: ../doc/lizard_Block_format.md
[lizard_Frame_format]: ../doc/lizard_Frame_format.md
#### Various Lizard builds
Files `lizard_common.h`, `lizard_compress*`, `lizard_parser_*.h`, `lizard_decompress*`, and `entropy\mem.h` are required in all circumstances.
To compile:
- Lizard_raw only with levels 10...29 : use the `-DLIZARD_NO_HUFFMAN` compiler flag
- Lizard_raw with levels 10...49 : include also all files from `entropy` directory
- Lizard_frame with levels 10...49 : `lizard_frame*` and all files from `entropy` and `xxhash` directories
#### Advanced API
A more complex `lizard_frame_static.h` is also provided.
It contains definitions which are not guaranteed to remain stable within future versions.
It must be used with static linking ***only***.
#### Using MinGW+MSYS to create DLL
DLL can be created using MinGW+MSYS with the `make liblizard` command.
This command creates `dll\liblizard.dll` and the import library `dll\liblizard.lib`.
The import library is only required with Visual C++.
The header files `lizard.h`, `lizardhc.h`, `lizard_frame.h` and the dynamic library
`dll\liblizard.dll` are required to compile a project using gcc/MinGW.
The dynamic library has to be added to linking options.
It means that if a project that uses Lizard consists of a single `test-dll.c`
file it should be compiled with `liblizard.lib`. For example:
```
gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblizard.dll
```
The compiled executable will require Lizard DLL which is available at `dll\liblizard.dll`.
#### Miscellaneous
Other files present in the directory are not source code. There are :
- LICENSE : contains the BSD license text
- Makefile : script to compile or install lizard library (static or dynamic)
- liblizard.pc.in : for pkg-config (make install)
- README.md : this file
#### License
All source material within __lib__ directory are BSD 2-Clause licensed.
See [LICENSE](LICENSE) for details.
The license is also repeated at the top of each source file.

19
contrib/lizard/lib/dll/liblizard.def Normal file
View File

@ -0,0 +1,19 @@
LIBRARY liblizard.dll
EXPORTS
Lizard_compress
Lizard_compressBound
Lizard_compress_continue
Lizard_compress_extState
Lizard_createStream
Lizard_createStreamDecode
Lizard_decompress_safe
Lizard_decompress_safe_continue
Lizard_decompress_safe_partial
Lizard_decompress_safe_usingDict
Lizard_freeStream
Lizard_freeStreamDecode
Lizard_loadDict
Lizard_resetStream
Lizard_saveDict
Lizard_setStreamDecode
Lizard_sizeofState

38
contrib/lizard/lib/entropy/README.md Normal file
View File

@ -0,0 +1,38 @@
New Generation Entropy library
==============================
The __lib__ directory contains several files, but you don't necessarily want them all.
Here is a detailed list, to help you decide which one you need :
#### Compulsory files
These files are required in all circumstances :
- __error_public.h__ : error list as enum
- __error_private.h__ : error management
- __mem.h__ : low level memory access routines
- __bitstream.h__ : generic read/write bitstream common to all entropy codecs
- __entropy_common.c__ : common functions needed for both compression and decompression
#### Finite State Entropy
This is the base codec required by other ones.
It implements a tANS variant, similar to arithmetic in compression performance, but much faster. Compression and decompression can be compiled independently.
- __fse.h__ : exposes interfaces
- __fse_compress.c__ : implements compression codec
- __fse_decompress.c__ : implements decompression codec
#### FSE 16-bits symbols version
This codec is able to encode alphabets of size > 256, using 2 bytes per symbol. It requires the base FSE codec to compile properly. Compression and decompression are merged in the same file.
- __fseU16.c__ implements the codec, while __fseU16.h__ exposes its interfaces.
#### Huffman codec
This is the fast huffman codec. It requires the base FSE codec to compress its headers. Compression and decompression can be compiled independently.
- __huf.h__ : exposes interfaces.
- __huf_compress.c__ : implements compression codec
- __huf_decompress.c__ : implements decompression codec

414
contrib/lizard/lib/entropy/bitstream.h Normal file
View File

@ -0,0 +1,414 @@
/* ******************************************************************
bitstream
Part of FSE library
header file (to include)
Copyright (C) 2013-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
****************************************************************** */
#ifndef BITSTREAM_H_MODULE
#define BITSTREAM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*
* This API consists of small unitary functions, which must be inlined for best performance.
* Since link-time-optimization is not available for all compilers,
* these functions are defined into a .h to be included.
*/
/*-****************************************
* Dependencies
******************************************/
#include "mem.h" /* unaligned access routines */
#include "error_private.h" /* error codes and messages */
/*=========================================
* Target specific
=========================================*/
#if defined(__BMI__) && defined(__GNUC__)
# include <immintrin.h> /* support for bextr (experimental) */
#endif
/*-******************************************
* bitStream encoding API (write forward)
********************************************/
/* bitStream can mix input from multiple sources.
* A critical property of these streams is that they encode and decode in **reverse** direction.
* So the first bit sequence you add will be the last to be read, like a LIFO stack.
*/
typedef struct
{
size_t bitContainer;
int bitPos;
char* startPtr;
char* ptr;
char* endPtr;
} BIT_CStream_t;
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
/* Start with initCStream, providing the size of buffer to write into.
* bitStream will never write outside of this buffer.
* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
*
* bits are first added to a local register.
* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
* Writing data into memory is an explicit operation, performed by the flushBits function.
* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
* After a flushBits, a maximum of 7 bits might still be stored into local register.
*
* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
*
* Last operation is to close the bitStream.
* The function returns the final size of CStream in bytes.
* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
*/
/*-********************************************
* bitStream decoding API (read backward)
**********************************************/
typedef struct
{
size_t bitContainer;
unsigned bitsConsumed;
const char* ptr;
const char* start;
} BIT_DStream_t;
typedef enum { BIT_DStream_unfinished = 0,
BIT_DStream_endOfBuffer = 1,
BIT_DStream_completed = 2,
BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
/* Start by invoking BIT_initDStream().
* A chunk of the bitStream is then stored into a local register.
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
* You can then retrieve bitFields stored into the local register, **in reverse order**.
* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
* Otherwise, it can be less than that, so proceed accordingly.
* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
*/
/*-****************************************
* unsafe API
******************************************/
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
/* unsafe version; does not check buffer overflow */
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
/* faster, but works only if nbBits >= 1 */
/*-**************************************************************
* Internal functions
****************************************************************/
MEM_STATIC unsigned BIT_highbit32 (register U32 val)
{
# if defined(_MSC_VER) /* Visual */
unsigned long r=0;
_BitScanReverse ( &r, val );
return (unsigned) r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
return 31 - __builtin_clz (val);
# else /* Software version */
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
# endif
}
/*===== Local Constants =====*/
static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
/*-**************************************************************
* bitStream encoding
****************************************************************/
/*! BIT_initCStream() :
* `dstCapacity` must be > sizeof(void*)
* @return : 0 if success,
otherwise an error code (can be tested using ERR_isError() ) */
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity)
{
bitC->bitContainer = 0;
bitC->bitPos = 0;
bitC->startPtr = (char*)startPtr;
bitC->ptr = bitC->startPtr;
bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr);
if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall);
return 0;
}
/*! BIT_addBits() :
can add up to 26 bits into `bitC`.
Does not check for register overflow ! */
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
{
bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
bitC->bitPos += nbBits;
}
/*! BIT_addBitsFast() :
* works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
{
bitC->bitContainer |= value << bitC->bitPos;
bitC->bitPos += nbBits;
}
/*! BIT_flushBitsFast() :
* unsafe version; does not check buffer overflow */
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
{
size_t const nbBytes = bitC->bitPos >> 3;
MEM_writeLEST(bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes;
bitC->bitPos &= 7;
bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
}
/*! BIT_flushBits() :
* safe version; check for buffer overflow, and prevents it.
* note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
{
size_t const nbBytes = bitC->bitPos >> 3;
MEM_writeLEST(bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes;
if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
bitC->bitPos &= 7;
bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
}
/*! BIT_closeCStream() :
* @return : size of CStream, in bytes,
or 0 if it could not fit into dstBuffer */
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
{
BIT_addBitsFast(bitC, 1, 1); /* endMark */
BIT_flushBits(bitC);
if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */
return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
}
/*-********************************************************
* bitStream decoding
**********************************************************/
/*! BIT_initDStream() :
* Initialize a BIT_DStream_t.
* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
* `srcSize` must be the *exact* size of the bitStream, in bytes.
* @return : size of stream (== srcSize) or an errorCode if a problem is detected
*/
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
{
if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
bitD->start = (const char*)srcBuffer;
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
bitD->bitContainer = MEM_readLEST(bitD->ptr);
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
} else {
bitD->start = (const char*)srcBuffer;
bitD->ptr = bitD->start;
bitD->bitContainer = *(const BYTE*)(bitD->start);
switch(srcSize)
{
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); // fallthrough
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); // fallthrough
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); // fallthrough
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; // fallthrough
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; // fallthrough
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; // fallthrough
default:;
}
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
}
return srcSize;
}
MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
{
return bitContainer >> start;
}
MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
{
#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008 /* experimental */
# if defined(__x86_64__)
if (sizeof(bitContainer)==8)
return _bextr_u64(bitContainer, start, nbBits);
else
# endif
return _bextr_u32(bitContainer, start, nbBits);
#else
return (bitContainer >> start) & BIT_mask[nbBits];
#endif
}
MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
{
return bitContainer & BIT_mask[nbBits];
}
/*! BIT_lookBits() :
* Provides next n bits from local register.
* local register is not modified.
* On 32-bits, maxNbBits==24.
* On 64-bits, maxNbBits==56.
* @return : value extracted
*/
MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
{
#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
#else
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
#endif
}
/*! BIT_lookBitsFast() :
* unsafe version; only works only if nbBits >= 1 */
MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
{
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
}
MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
{
bitD->bitsConsumed += nbBits;
}
/*! BIT_readBits() :
* Read (consume) next n bits from local register and update.
* Pay attention to not read more than nbBits contained into local register.
* @return : extracted value.
*/
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
{
size_t const value = BIT_lookBits(bitD, nbBits);
BIT_skipBits(bitD, nbBits);
return value;
}
/*! BIT_readBitsFast() :
* unsafe version; only works only if nbBits >= 1 */
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
{
size_t const value = BIT_lookBitsFast(bitD, nbBits);
BIT_skipBits(bitD, nbBits);
return value;
}
/*! BIT_reloadDStream() :
* Refill `bitD` from buffer previously set in BIT_initDStream() .
* This function is safe, it guarantees it will not read beyond src buffer.
* @return : status of `BIT_DStream_t` internal register.
if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
{
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
return BIT_DStream_overflow;
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
bitD->ptr -= bitD->bitsConsumed >> 3;
bitD->bitsConsumed &= 7;
bitD->bitContainer = MEM_readLEST(bitD->ptr);
return BIT_DStream_unfinished;
}
if (bitD->ptr == bitD->start) {
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
return BIT_DStream_completed;
}
{ U32 nbBytes = bitD->bitsConsumed >> 3;
BIT_DStream_status result = BIT_DStream_unfinished;
if (bitD->ptr - nbBytes < bitD->start) {
nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
result = BIT_DStream_endOfBuffer;
}
bitD->ptr -= nbBytes;
bitD->bitsConsumed -= nbBytes*8;
bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
return result;
}
}
/*! BIT_endOfDStream() :
* @return Tells if DStream has exactly reached its end (all bits consumed).
*/
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
{
return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
}
#if defined (__cplusplus)
}
#endif
#endif /* BITSTREAM_H_MODULE */

231
contrib/lizard/lib/entropy/entropy_common.c Normal file
View File

@ -0,0 +1,231 @@
/*
Common functions of New Generation Entropy library
Copyright (C) 2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
*************************************************************************** */
/* *************************************
* Dependencies
***************************************/
#include "mem.h"
#include "error_private.h" /* ERR_*, ERROR */
#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */
#include "huf.h"
/*-****************************************
* Version
******************************************/
unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
/*-****************************************
* FSE Error Management
******************************************/
unsigned FSE_isError(size_t code) { return ERR_isError(code); }
const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
/* **************************************************************
* HUF Error Management
****************************************************************/
unsigned HUF_isError(size_t code) { return ERR_isError(code); }
const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
/*-**************************************************************
* FSE NCount encoding-decoding
****************************************************************/
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
const void* headerBuffer, size_t hbSize)
{
const BYTE* const istart = (const BYTE*) headerBuffer;
const BYTE* const iend = istart + hbSize;
const BYTE* ip = istart;
int nbBits;
int remaining;
int threshold;
U32 bitStream;
int bitCount;
unsigned charnum = 0;
int previous0 = 0;
if (hbSize < 4) return ERROR(srcSize_wrong);
bitStream = MEM_readLE32(ip);
nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
bitStream >>= 4;
bitCount = 4;
*tableLogPtr = nbBits;
remaining = (1<<nbBits)+1;
threshold = 1<<nbBits;
nbBits++;
while ((remaining>1) & (charnum<=*maxSVPtr)) {
if (previous0) {
unsigned n0 = charnum;
while ((bitStream & 0xFFFF) == 0xFFFF) {
n0 += 24;
if (ip < iend-5) {
ip += 2;
bitStream = MEM_readLE32(ip) >> bitCount;
} else {
bitStream >>= 16;
bitCount += 16;
} }
while ((bitStream & 3) == 3) {
n0 += 3;
bitStream >>= 2;
bitCount += 2;
}
n0 += bitStream & 3;
bitCount += 2;
if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
while (charnum < n0) normalizedCounter[charnum++] = 0;
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
ip += bitCount>>3;
bitCount &= 7;
bitStream = MEM_readLE32(ip) >> bitCount;
} else {
bitStream >>= 2;
} }
{ int const max = (2*threshold-1) - remaining;
int count;
if ((bitStream & (threshold-1)) < (U32)max) {
count = bitStream & (threshold-1);
bitCount += nbBits-1;
} else {
count = bitStream & (2*threshold-1);
if (count >= threshold) count -= max;
bitCount += nbBits;
}
count--; /* extra accuracy */
remaining -= count < 0 ? -count : count; /* -1 means +1 */
normalizedCounter[charnum++] = (short)count;
previous0 = !count;
while (remaining < threshold) {
nbBits--;
threshold >>= 1;
}
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
ip += bitCount>>3;
bitCount &= 7;
} else {
bitCount -= (int)(8 * (iend - 4 - ip));
ip = iend - 4;
}
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
} } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
if (remaining != 1) return ERROR(corruption_detected);
if (bitCount > 32) return ERROR(corruption_detected);
*maxSVPtr = charnum-1;
ip += (bitCount+7)>>3;
return ip-istart;
}
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
`rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
@return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
*/
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize)
{
U32 weightTotal;
const BYTE* ip = (const BYTE*) src;
size_t iSize;
size_t oSize;
if (!srcSize) return ERROR(srcSize_wrong);
iSize = ip[0];
/* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
oSize = iSize - 127;
iSize = ((oSize+1)/2);
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
if (oSize >= hwSize) return ERROR(corruption_detected);
ip += 1;
{ U32 n;
for (n=0; n<oSize; n+=2) {
huffWeight[n] = ip[n/2] >> 4;
huffWeight[n+1] = ip[n/2] & 15;
} } }
else { /* header compressed with FSE (normal case) */
FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */
if (FSE_isError(oSize)) return oSize;
}
/* collect weight stats */
memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
weightTotal = 0;
{ U32 n; for (n=0; n<oSize; n++) {
if (huffWeight[n] >= HUF_TABLELOG_MAX) return ERROR(corruption_detected);
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
} }
if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
*tableLogPtr = tableLog;
/* determine last weight */
{ U32 const total = 1 << tableLog;
U32 const rest = total - weightTotal;
U32 const verif = 1 << BIT_highbit32(rest);
U32 const lastWeight = BIT_highbit32(rest) + 1;
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
huffWeight[oSize] = (BYTE)lastWeight;
rankStats[lastWeight]++;
} }
/* check tree construction validity */
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
/* results */
*nbSymbolsPtr = (U32)(oSize+1);
return iSize+1;
}

115
contrib/lizard/lib/entropy/error_private.h Normal file
View File

@ -0,0 +1,115 @@
/* ******************************************************************
Error codes and messages
Copyright (C) 2013-2016, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Homepage : http://www.zstd.net
****************************************************************** */
/* Note : this module is expected to remain private, do not expose it */
#ifndef ERROR_H_MODULE
#define ERROR_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* Dependencies
******************************************/
#include <stddef.h> /* size_t */
#include "error_public.h" /* enum list */
/* ****************************************
* Compiler-specific
******************************************/
#if defined(__GNUC__)
# define ERR_STATIC static __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define ERR_STATIC static inline
#elif defined(_MSC_VER)
# define ERR_STATIC static __inline
#else
# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/*-****************************************
* Customization (error_public.h)
******************************************/
typedef FSE_ErrorCode ERR_enum;
#define PREFIX(name) FSE_error_##name
/*-****************************************
* Error codes handling
******************************************/
#ifdef ERROR
# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
#endif
#define ERROR(name) ((size_t)-PREFIX(name))
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
/*-****************************************
* Error Strings
******************************************/
ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
{
static const char* notErrorCode = "Unspecified error code";
switch( code )
{
case PREFIX(no_error): return "No error detected";
case PREFIX(GENERIC): return "Error (generic)";
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size incorrect";
case PREFIX(corruption_detected): return "Corrupted block detected";
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(maxCode):
default: return notErrorCode;
}
}
ERR_STATIC const char* ERR_getErrorName(size_t code)
{
return ERR_getErrorString(ERR_getErrorCode(code));
}
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_H_MODULE */

64
contrib/lizard/lib/entropy/error_public.h Normal file
View File

@ -0,0 +1,64 @@
/* ******************************************************************
Error codes list
Copyright (C) 2016, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
#ifndef ERROR_PUBLIC_H_MODULE
#define ERROR_PUBLIC_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* error codes list
******************************************/
typedef enum {
FSE_error_no_error,
FSE_error_GENERIC,
FSE_error_dstSize_tooSmall,
FSE_error_srcSize_wrong,
FSE_error_corruption_detected,
FSE_error_tableLog_tooLarge,
FSE_error_maxSymbolValue_tooLarge,
FSE_error_maxSymbolValue_tooSmall,
FSE_error_maxCode
} FSE_ErrorCode;
/* note : compare with size_t function results using FSE_getError() */
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_PUBLIC_H_MODULE */

694
contrib/lizard/lib/entropy/fse.h Normal file
View File

@ -0,0 +1,694 @@
/* ******************************************************************
FSE : Finite State Entropy codec
Public Prototypes declaration
Copyright (C) 2013-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
****************************************************************** */
#ifndef FSE_H
#define FSE_H
#if defined (__cplusplus)
extern "C" {
#endif
/*-*****************************************
* Dependencies
******************************************/
#include <stddef.h> /* size_t, ptrdiff_t */
/*-*****************************************
* FSE_PUBLIC_API : control library symbols visibility
******************************************/
#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
# define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
# define FSE_PUBLIC_API __declspec(dllexport)
#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define FSE_PUBLIC_API
#endif
/*------ Version ------*/
#define FSE_VERSION_MAJOR 0
#define FSE_VERSION_MINOR 9
#define FSE_VERSION_RELEASE 0
#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
#define FSE_QUOTE(str) #str
#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
/*-****************************************
* FSE simple functions
******************************************/
/*! FSE_compress() :
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
@return : size of compressed data (<= dstCapacity).
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
*/
FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/*! FSE_decompress():
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
into already allocated destination buffer 'dst', of size 'dstCapacity'.
@return : size of regenerated data (<= maxDstSize),
or an error code, which can be tested using FSE_isError() .
** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
Why ? : making this distinction requires a header.
Header management is intentionally delegated to the user layer, which can better manage special cases.
*/
FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity,
const void* cSrc, size_t cSrcSize);
/*-*****************************************
* Tool functions
******************************************/
FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */
/* Error Management */
FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
/*-*****************************************
* FSE advanced functions
******************************************/
/*! FSE_compress2() :
Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
Both parameters can be defined as '0' to mean : use default value
@return : size of compressed data
Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
if FSE_isError(return), it's an error code.
*/
FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
/*-*****************************************
* FSE detailed API
******************************************/
/*!
FSE_compress() does the following:
1. count symbol occurrence from source[] into table count[]
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
3. save normalized counters to memory buffer using writeNCount()
4. build encoding table 'CTable' from normalized counters
5. encode the data stream using encoding table 'CTable'
FSE_decompress() does the following:
1. read normalized counters with readNCount()
2. build decoding table 'DTable' from normalized counters
3. decode the data stream using decoding table 'DTable'
The following API allows targeting specific sub-functions for advanced tasks.
For example, it's possible to compress several blocks using the same 'CTable',
or to save and provide normalized distribution using external method.
*/
/* *** COMPRESSION *** */
/*! FSE_count():
Provides the precise count of each byte within a table 'count'.
'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
*maxSymbolValuePtr will be updated if detected smaller than initial value.
@return : the count of the most frequent symbol (which is not identified).
if return == srcSize, there is only one symbol.
Can also return an error code, which can be tested with FSE_isError(). */
FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
/*! FSE_optimalTableLog():
dynamically downsize 'tableLog' when conditions are met.
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
@return : recommended tableLog (necessarily <= 'maxTableLog') */
FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
/*! FSE_normalizeCount():
normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
@return : tableLog,
or an errorCode, which can be tested using FSE_isError() */
FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
/*! FSE_NCountWriteBound():
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
Typically useful for allocation purpose. */
FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_writeNCount():
Compactly save 'normalizedCounter' into 'buffer'.
@return : size of the compressed table,
or an errorCode, which can be tested using FSE_isError(). */
FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! Constructor and Destructor of FSE_CTable.
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct);
/*! FSE_buildCTable():
Builds `ct`, which must be already allocated, using FSE_createCTable().
@return : 0, or an errorCode, which can be tested using FSE_isError() */
FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_compress_usingCTable():
Compress `src` using `ct` into `dst` which must be already allocated.
@return : size of compressed data (<= `dstCapacity`),
or 0 if compressed data could not fit into `dst`,
or an errorCode, which can be tested using FSE_isError() */
FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
/*!
Tutorial :
----------
The first step is to count all symbols. FSE_count() does this job very fast.
Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
FSE_count() will return the number of occurrence of the most frequent symbol.
This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
The next step is to normalize the frequencies.
FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
You can use 'tableLog'==0 to mean "use default tableLog value".
If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
The result of FSE_normalizeCount() will be saved into a table,
called 'normalizedCounter', which is a table of signed short.
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
The return value is tableLog if everything proceeded as expected.
It is 0 if there is a single symbol within distribution.
If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
'buffer' must be already allocated.
For guaranteed success, buffer size must be at least FSE_headerBound().
The result of the function is the number of bytes written into 'buffer'.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
'normalizedCounter' can then be used to create the compression table 'CTable'.
The space required by 'CTable' must be already allocated, using FSE_createCTable().
You can then use FSE_buildCTable() to fill 'CTable'.
If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.
If it returns '0', compressed data could not fit into 'dst'.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
*/
/* *** DECOMPRESSION *** */
/*! FSE_readNCount():
Read compactly saved 'normalizedCounter' from 'rBuffer'.
@return : size read from 'rBuffer',
or an errorCode, which can be tested using FSE_isError().
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
/*! Constructor and Destructor of FSE_DTable.
Note that its size depends on 'tableLog' */
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt);
/*! FSE_buildDTable():
Builds 'dt', which must be already allocated, using FSE_createDTable().
return : 0, or an errorCode, which can be tested using FSE_isError() */
FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_decompress_usingDTable():
Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
into `dst` which must be already allocated.
@return : size of regenerated data (necessarily <= `dstCapacity`),
or an errorCode, which can be tested using FSE_isError() */
FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
/*!
Tutorial :
----------
(Note : these functions only decompress FSE-compressed blocks.
If block is uncompressed, use memcpy() instead
If block is a single repeated byte, use memset() instead )
The first step is to obtain the normalized frequencies of symbols.
This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
or size the table to handle worst case situations (typically 256).
FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
If there is an error, the function will return an error code, which can be tested using FSE_isError().
The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
This is performed by the function FSE_buildDTable().
The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
If there is an error, the function will return an error code, which can be tested using FSE_isError().
`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().
`cSrcSize` must be strictly correct, otherwise decompression will fail.
FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
*/
#ifdef FSE_STATIC_LINKING_ONLY
/* *** Dependency *** */
#include "bitstream.h"
/* *****************************************
* Static allocation
*******************************************/
/* FSE buffer bounds */
#define FSE_NCOUNTBOUND 512
#define FSE_BLOCKBOUND(size) (size + (size>>7))
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
/* *****************************************
* FSE advanced API
*******************************************/
/* FSE_count_wksp() :
* Same as FSE_count(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= `1024` unsigned
*/
size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize, unsigned* workSpace);
/** FSE_countFast() :
* same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr
*/
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
/* FSE_countFast_wksp() :
* Same as FSE_countFast(), but using an externally provided scratch buffer.
* `workSpace` must be a table of minimum `1024` unsigned
*/
size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace);
/*! FSE_count_simple
* Same as FSE_countFast(), but does not use any additional memory (not even on stack).
* This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`).
*/
size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
/**< same as FSE_optimalTableLog(), which used `minus==2` */
/* FSE_compress_wksp() :
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
* FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
*/
#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + (1<<((maxTableLog>2)?(maxTableLog-2):0)) )
size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
/* FSE_buildCTable_wksp() :
* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
* `wkspSize` must be >= `(1<<tableLog)`.
*/
size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog);
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */
/* *****************************************
* FSE symbol compression API
*******************************************/
/*!
This API consists of small unitary functions, which highly benefit from being inlined.
Hence their body are included in next section.
*/
typedef struct {
ptrdiff_t value;
const void* stateTable;
const void* symbolTT;
unsigned stateLog;
} FSE_CState_t;
static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
/**<
These functions are inner components of FSE_compress_usingCTable().
They allow the creation of custom streams, mixing multiple tables and bit sources.
A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
So the first symbol you will encode is the last you will decode, like a LIFO stack.
You will need a few variables to track your CStream. They are :
FSE_CTable ct; // Provided by FSE_buildCTable()
BIT_CStream_t bitStream; // bitStream tracking structure
FSE_CState_t state; // State tracking structure (can have several)
The first thing to do is to init bitStream and state.
size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
FSE_initCState(&state, ct);
Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
You can then encode your input data, byte after byte.
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
Remember decoding will be done in reverse direction.
FSE_encodeByte(&bitStream, &state, symbol);
At any time, you can also add any bit sequence.
Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
BIT_addBits(&bitStream, bitField, nbBits);
The above methods don't commit data to memory, they just store it into local register, for speed.
Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
Writing data to memory is a manual operation, performed by the flushBits function.
BIT_flushBits(&bitStream);
Your last FSE encoding operation shall be to flush your last state value(s).
FSE_flushState(&bitStream, &state);
Finally, you must close the bitStream.
The function returns the size of CStream in bytes.
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
size_t size = BIT_closeCStream(&bitStream);
*/
/* *****************************************
* FSE symbol decompression API
*******************************************/
typedef struct {
size_t state;
const void* table; /* precise table may vary, depending on U16 */
} FSE_DState_t;
static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
/**<
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
You will decode FSE-encoded symbols from the bitStream,
and also any other bitFields you put in, **in reverse order**.
You will need a few variables to track your bitStream. They are :
BIT_DStream_t DStream; // Stream context
FSE_DState_t DState; // State context. Multiple ones are possible
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
The first thing to do is to init the bitStream.
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
You should then retrieve your initial state(s)
(in reverse flushing order if you have several ones) :
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
You can then decode your data, symbol after symbol.
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
Note : maximum allowed nbBits is 25, for 32-bits compatibility
size_t bitField = BIT_readBits(&DStream, nbBits);
All above operations only read from local register (which size depends on size_t).
Refueling the register from memory is manually performed by the reload method.
endSignal = FSE_reloadDStream(&DStream);
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
BIT_DStream_unfinished : there is still some data left into the DStream.
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
to properly detect the exact end of stream.
After each decoded symbol, check if DStream is fully consumed using this simple test :
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
Checking if DStream has reached its end is performed by :
BIT_endOfDStream(&DStream);
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
FSE_endOfDState(&DState);
*/
/* *****************************************
* FSE unsafe API
*******************************************/
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
/* *****************************************
* Implementation of inlined functions
*******************************************/
typedef struct {
int deltaFindState;
U32 deltaNbBits;
} FSE_symbolCompressionTransform; /* total 8 bytes */
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
{
const void* ptr = ct;
const U16* u16ptr = (const U16*) ptr;
const U32 tableLog = MEM_read16(ptr);
statePtr->value = (ptrdiff_t)1<<tableLog;
statePtr->stateTable = u16ptr+2;
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
statePtr->stateLog = tableLog;
}
/*! FSE_initCState2() :
* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
* uses the smallest state value possible, saving the cost of this symbol */
MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
{
FSE_initCState(statePtr, ct);
{ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
const U16* stateTable = (const U16*)(statePtr->stateTable);
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
}
}
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
{
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
const U16* const stateTable = (const U16*)(statePtr->stateTable);
U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
BIT_addBits(bitC, statePtr->value, nbBitsOut);
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
}
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
{
BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
BIT_flushBits(bitC);
}
/* ====== Decompression ====== */
typedef struct {
U16 tableLog;
U16 fastMode;
} FSE_DTableHeader; /* sizeof U32 */
typedef struct
{
unsigned short newState;
unsigned char symbol;
unsigned char nbBits;
} FSE_decode_t; /* size == U32 */
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
{
const void* ptr = dt;
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1;
}
MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
return DInfo.symbol;
}
MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
}
MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
BYTE const symbol = DInfo.symbol;
size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
return symbol;
}
/*! FSE_decodeSymbolFast() :
unsafe, only works if no symbol has a probability > 50% */
MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
BYTE const symbol = DInfo.symbol;
size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
return symbol;
}
MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
{
return DStatePtr->state == 0;
}
#ifndef FSE_COMMONDEFS_ONLY
/* **************************************************************
* Tuning parameters
****************************************************************/
/*!MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
#ifndef FSE_MAX_MEMORY_USAGE
# define FSE_MAX_MEMORY_USAGE 14
#endif
#ifndef FSE_DEFAULT_MEMORY_USAGE
# define FSE_DEFAULT_MEMORY_USAGE 13
#endif
/*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized.
* Required for proper stack allocation */
#ifndef FSE_MAX_SYMBOL_VALUE
# define FSE_MAX_SYMBOL_VALUE 255
#endif
/* **************************************************************
* template functions type & suffix
****************************************************************/
#define FSE_FUNCTION_TYPE BYTE
#define FSE_FUNCTION_EXTENSION
#define FSE_DECODE_TYPE FSE_decode_t
#endif /* !FSE_COMMONDEFS_ONLY */
/* ***************************************************************
* Constants
*****************************************************************/
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
#define FSE_MIN_TABLELOG 5
#define FSE_TABLELOG_ABSOLUTE_MAX 15
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
#endif
#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
#endif /* FSE_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
#endif /* FSE_H */

848
contrib/lizard/lib/entropy/fse_compress.c Normal file
View File

@ -0,0 +1,848 @@
/* ******************************************************************
FSE : Finite State Entropy encoder
Copyright (C) 2013-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
/* **************************************************************
* Compiler specifics
****************************************************************/
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif
/* **************************************************************
* Includes
****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */
#include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */
#include "bitstream.h"
#define FSE_STATIC_LINKING_ONLY
#include "fse.h"
/* **************************************************************
* Error Management
****************************************************************/
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/* **************************************************************
* Templates
****************************************************************/
/*
designed to be included
for type-specific functions (template emulation in C)
Objective is to write these functions only once, for improved maintenance
*/
/* safety checks */
#ifndef FSE_FUNCTION_EXTENSION
# error "FSE_FUNCTION_EXTENSION must be defined"
#endif
#ifndef FSE_FUNCTION_TYPE
# error "FSE_FUNCTION_TYPE must be defined"
#endif
/* Function names */
#define FSE_CAT(X,Y) X##Y
#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
/* Function templates */
/* FSE_buildCTable_wksp() :
* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
* wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
* workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
*/
size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
{
U32 const tableSize = 1 << tableLog;
U32 const tableMask = tableSize - 1;
void* const ptr = ct;
U16* const tableU16 = ( (U16*) ptr) + 2;
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
U32 const step = FSE_TABLESTEP(tableSize);
U32 cumul[FSE_MAX_SYMBOL_VALUE+2];
FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace;
U32 highThreshold = tableSize-1;
/* CTable header */
if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge);
tableU16[-2] = (U16) tableLog;
tableU16[-1] = (U16) maxSymbolValue;
/* For explanations on how to distribute symbol values over the table :
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
/* symbol start positions */
{ U32 u;
cumul[0] = 0;
for (u=1; u<=maxSymbolValue+1; u++) {
if (normalizedCounter[u-1]==-1) { /* Low proba symbol */
cumul[u] = cumul[u-1] + 1;
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
} else {
cumul[u] = cumul[u-1] + normalizedCounter[u-1];
} }
cumul[maxSymbolValue+1] = tableSize+1;
}
/* Spread symbols */
{ U32 position = 0;
U32 symbol;
for (symbol=0; symbol<=maxSymbolValue; symbol++) {
int nbOccurences;
for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++) {
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
position = (position + step) & tableMask;
while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */
} }
if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */
}
/* Build table */
{ U32 u; for (u=0; u<tableSize; u++) {
FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */
} }
/* Build Symbol Transformation Table */
{ unsigned total = 0;
unsigned s;
for (s=0; s<=maxSymbolValue; s++) {
switch (normalizedCounter[s])
{
case 0: break;
case -1:
case 1:
symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog);
symbolTT[s].deltaFindState = total - 1;
total ++;
break;
default :
{
U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1);
U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
symbolTT[s].deltaFindState = total - normalizedCounter[s];
total += normalizedCounter[s];
} } } }
return 0;
}
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, tableSymbol, sizeof(tableSymbol));
}
#ifndef FSE_COMMONDEFS_ONLY
/*-**************************************************************
* FSE NCount encoding-decoding
****************************************************************/
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
{
size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
}
static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
unsigned writeIsSafe)
{
BYTE* const ostart = (BYTE*) header;
BYTE* out = ostart;
BYTE* const oend = ostart + headerBufferSize;
int nbBits;
const int tableSize = 1 << tableLog;
int remaining;
int threshold;
U32 bitStream;
int bitCount;
unsigned charnum = 0;
int previous0 = 0;
bitStream = 0;
bitCount = 0;
/* Table Size */
bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
bitCount += 4;
/* Init */
remaining = tableSize+1; /* +1 for extra accuracy */
threshold = tableSize;
nbBits = tableLog+1;
while (remaining>1) { /* stops at 1 */
if (previous0) {
unsigned start = charnum;
while (!normalizedCounter[charnum]) charnum++;
while (charnum >= start+24) {
start+=24;
bitStream += 0xFFFFU << bitCount;
if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE) bitStream;
out[1] = (BYTE)(bitStream>>8);
out+=2;
bitStream>>=16;
}
while (charnum >= start+3) {
start+=3;
bitStream += 3 << bitCount;
bitCount += 2;
}
bitStream += (charnum-start) << bitCount;
bitCount += 2;
if (bitCount>16) {
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8);
out += 2;
bitStream >>= 16;
bitCount -= 16;
} }
{ int count = normalizedCounter[charnum++];
int const max = (2*threshold-1)-remaining;
remaining -= count < 0 ? -count : count;
count++; /* +1 for extra accuracy */
if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
bitStream += count << bitCount;
bitCount += nbBits;
bitCount -= (count<max);
previous0 = (count==1);
if (remaining<1) return ERROR(GENERIC);
while (remaining<threshold) nbBits--, threshold>>=1;
}
if (bitCount>16) {
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8);
out += 2;
bitStream >>= 16;
bitCount -= 16;
} }
/* flush remaining bitStream */
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8);
out+= (bitCount+7) /8;
if (charnum > maxSymbolValue + 1) return ERROR(GENERIC);
return (out-ostart);
}
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC); /* Unsupported */
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */
if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
}
/*-**************************************************************
* Counting histogram
****************************************************************/
/*! FSE_count_simple
This function counts byte values within `src`, and store the histogram into table `count`.
It doesn't use any additional memory.
But this function is unsafe : it doesn't check that all values within `src` can fit into `count`.
For this reason, prefer using a table `count` with 256 elements.
@return : count of most numerous element
*/
size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
const BYTE* const end = ip + srcSize;
unsigned maxSymbolValue = *maxSymbolValuePtr;
unsigned max=0;
memset(count, 0, (maxSymbolValue+1)*sizeof(*count));
if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; }
while (ip<end) count[*ip++]++;
while (!count[maxSymbolValue]) maxSymbolValue--;
*maxSymbolValuePtr = maxSymbolValue;
{ U32 s; for (s=0; s<=maxSymbolValue; s++) if (count[s] > max) max = count[s]; }
return (size_t)max;
}
/* FSE_count_parallel_wksp() :
* Same as FSE_count_parallel(), but using an externally provided scratch buffer.
* `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */
static size_t FSE_count_parallel_wksp(
unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize,
unsigned checkMax, unsigned* const workSpace)
{
const BYTE* ip = (const BYTE*)source;
const BYTE* const iend = ip+sourceSize;
unsigned maxSymbolValue = *maxSymbolValuePtr;
unsigned max=0;
U32* const Counting1 = workSpace;
U32* const Counting2 = Counting1 + 256;
U32* const Counting3 = Counting2 + 256;
U32* const Counting4 = Counting3 + 256;
memset(Counting1, 0, 4*256*sizeof(unsigned));
/* safety checks */
if (!sourceSize) {
memset(count, 0, maxSymbolValue + 1);
*maxSymbolValuePtr = 0;
return 0;
}
if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */
/* by stripes of 16 bytes */
{ U32 cached = MEM_read32(ip); ip += 4;
while (ip < iend-15) {
U32 c = cached; cached = MEM_read32(ip); ip += 4;
Counting1[(BYTE) c ]++;
Counting2[(BYTE)(c>>8) ]++;
Counting3[(BYTE)(c>>16)]++;
Counting4[ c>>24 ]++;
c = cached; cached = MEM_read32(ip); ip += 4;
Counting1[(BYTE) c ]++;
Counting2[(BYTE)(c>>8) ]++;
Counting3[(BYTE)(c>>16)]++;
Counting4[ c>>24 ]++;
c = cached; cached = MEM_read32(ip); ip += 4;
Counting1[(BYTE) c ]++;
Counting2[(BYTE)(c>>8) ]++;
Counting3[(BYTE)(c>>16)]++;
Counting4[ c>>24 ]++;
c = cached; cached = MEM_read32(ip); ip += 4;
Counting1[(BYTE) c ]++;
Counting2[(BYTE)(c>>8) ]++;
Counting3[(BYTE)(c>>16)]++;
Counting4[ c>>24 ]++;
}
ip-=4;
}
/* finish last symbols */
while (ip<iend) Counting1[*ip++]++;
if (checkMax) { /* verify stats will fit into destination table */
U32 s; for (s=255; s>maxSymbolValue; s--) {
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall);
} }
{ U32 s; for (s=0; s<=maxSymbolValue; s++) {
count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
if (count[s] > max) max = count[s];
} }
while (!count[maxSymbolValue]) maxSymbolValue--;
*maxSymbolValuePtr = maxSymbolValue;
return (size_t)max;
}
/* FSE_countFast_wksp() :
* Same as FSE_countFast(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= `1024` unsigned */
size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize, unsigned* workSpace)
{
if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize);
return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace);
}
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize)
{
unsigned tmpCounters[1024];
return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters);
}
/* FSE_count_wksp() :
* Same as FSE_count(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= `1024` unsigned */
size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize, unsigned* workSpace)
{
if (*maxSymbolValuePtr < 255)
return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace);
*maxSymbolValuePtr = 255;
return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace);
}
size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize)
{
unsigned tmpCounters[1024];
return FSE_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters);
}
/*-**************************************************************
* FSE Compression Code
****************************************************************/
/*! FSE_sizeof_CTable() :
FSE_CTable is a variable size structure which contains :
`U16 tableLog;`
`U16 maxSymbolValue;`
`U16 nextStateNumber[1 << tableLog];` // This size is variable
`FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable
Allocation is manual (C standard does not support variable-size structures).
*/
size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog)
{
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
return FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
}
FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
{
size_t size;
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
return (FSE_CTable*)malloc(size);
}
void FSE_freeCTable (FSE_CTable* ct) { free(ct); }
/* provides the minimum logSize to safely represent a distribution */
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
{
U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1;
U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
return minBits;
}
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
{
U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
U32 tableLog = maxTableLog;
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
return tableLog;
}
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
{
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
}
/* Secondary normalization method.
To be used when primary method fails. */
static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
{
U32 s;
U32 distributed = 0;
U32 ToDistribute;
/* Init */
U32 const lowThreshold = (U32)(total >> tableLog);
U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
for (s=0; s<=maxSymbolValue; s++) {
if (count[s] == 0) {
norm[s]=0;
continue;
}
if (count[s] <= lowThreshold) {
norm[s] = -1;
distributed++;
total -= count[s];
continue;
}
if (count[s] <= lowOne) {
norm[s] = 1;
distributed++;
total -= count[s];
continue;
}
norm[s]=-2;
}
ToDistribute = (1 << tableLog) - distributed;
if ((total / ToDistribute) > lowOne) {
/* risk of rounding to zero */
lowOne = (U32)((total * 3) / (ToDistribute * 2));
for (s=0; s<=maxSymbolValue; s++) {
if ((norm[s] == -2) && (count[s] <= lowOne)) {
norm[s] = 1;
distributed++;
total -= count[s];
continue;
} }
ToDistribute = (1 << tableLog) - distributed;
}
if (distributed == maxSymbolValue+1) {
/* all values are pretty poor;
probably incompressible data (should have already been detected);
find max, then give all remaining points to max */
U32 maxV = 0, maxC = 0;
for (s=0; s<=maxSymbolValue; s++)
if (count[s] > maxC) maxV=s, maxC=count[s];
norm[maxV] += (short)ToDistribute;
return 0;
}
{ U64 const vStepLog = 62 - tableLog;
U64 const mid = (1ULL << (vStepLog-1)) - 1;
U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */
U64 tmpTotal = mid;
for (s=0; s<=maxSymbolValue; s++) {
if (norm[s]==-2) {
U64 const end = tmpTotal + (count[s] * rStep);
U32 const sStart = (U32)(tmpTotal >> vStepLog);
U32 const sEnd = (U32)(end >> vStepLog);
U32 const weight = sEnd - sStart;
if (weight < 1)
return ERROR(GENERIC);
norm[s] = (short)weight;
tmpTotal = end;
} } }
return 0;
}
size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
const unsigned* count, size_t total,
unsigned maxSymbolValue)
{
/* Sanity checks */
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */
if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
{ U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
U64 const scale = 62 - tableLog;
U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */
U64 const vStep = 1ULL<<(scale-20);
int stillToDistribute = 1<<tableLog;
unsigned s;
unsigned largest=0;
short largestP=0;
U32 lowThreshold = (U32)(total >> tableLog);
for (s=0; s<=maxSymbolValue; s++) {
if (count[s] == total) return 0; /* rle special case */
if (count[s] == 0) { normalizedCounter[s]=0; continue; }
if (count[s] <= lowThreshold) {
normalizedCounter[s] = -1;
stillToDistribute--;
} else {
short proba = (short)((count[s]*step) >> scale);
if (proba<8) {
U64 restToBeat = vStep * rtbTable[proba];
proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat;
}
if (proba > largestP) largestP=proba, largest=s;
normalizedCounter[s] = proba;
stillToDistribute -= proba;
} }
if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
/* corner case, need another normalization method */
size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
if (FSE_isError(errorCode)) return errorCode;
}
else normalizedCounter[largest] += (short)stillToDistribute;
}
#if 0
{ /* Print Table (debug) */
U32 s;
U32 nTotal = 0;
for (s=0; s<=maxSymbolValue; s++)
printf("%3i: %4i \n", s, normalizedCounter[s]);
for (s=0; s<=maxSymbolValue; s++)
nTotal += abs(normalizedCounter[s]);
if (nTotal != (1U<<tableLog))
printf("Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog);
getchar();
}
#endif
return tableLog;
}
/* fake FSE_CTable, for raw (uncompressed) input */
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
{
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask;
void* const ptr = ct;
U16* const tableU16 = ( (U16*) ptr) + 2;
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
unsigned s;
/* Sanity checks */
if (nbBits < 1) return ERROR(GENERIC); /* min size */
/* header */
tableU16[-2] = (U16) nbBits;
tableU16[-1] = (U16) maxSymbolValue;
/* Build table */
for (s=0; s<tableSize; s++)
tableU16[s] = (U16)(tableSize + s);
/* Build Symbol Transformation Table */
{ const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
for (s=0; s<=maxSymbolValue; s++) {
symbolTT[s].deltaNbBits = deltaNbBits;
symbolTT[s].deltaFindState = s-1;
} }
return 0;
}
/* fake FSE_CTable, for rle input (always same symbol) */
size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
{
void* ptr = ct;
U16* tableU16 = ( (U16*) ptr) + 2;
void* FSCTptr = (U32*)ptr + 2;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr;
/* header */
tableU16[-2] = (U16) 0;
tableU16[-1] = (U16) symbolValue;
/* Build table */
tableU16[0] = 0;
tableU16[1] = 0; /* just in case */
/* Build Symbol Transformation Table */
symbolTT[symbolValue].deltaNbBits = 0;
symbolTT[symbolValue].deltaFindState = 0;
return 0;
}
static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
const void* src, size_t srcSize,
const FSE_CTable* ct, const unsigned fast)
{
const BYTE* const istart = (const BYTE*) src;
const BYTE* const iend = istart + srcSize;
const BYTE* ip=iend;
BIT_CStream_t bitC;
FSE_CState_t CState1, CState2;
/* init */
if (srcSize <= 2) return 0;
{ size_t const initError = BIT_initCStream(&bitC, dst, dstSize);
if (FSE_isError(initError)) return 0; /* not enough space available to write a bitstream */ }
#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
if (srcSize & 1) {
FSE_initCState2(&CState1, ct, *--ip);
FSE_initCState2(&CState2, ct, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
FSE_FLUSHBITS(&bitC);
} else {
FSE_initCState2(&CState2, ct, *--ip);
FSE_initCState2(&CState1, ct, *--ip);
}
/* join to mod 4 */
srcSize -= 2;
if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */
FSE_encodeSymbol(&bitC, &CState2, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
FSE_FLUSHBITS(&bitC);
}
/* 2 or 4 encoding per loop */
while ( ip>istart ) {
FSE_encodeSymbol(&bitC, &CState2, *--ip);
if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */
FSE_FLUSHBITS(&bitC);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */
FSE_encodeSymbol(&bitC, &CState2, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
}
FSE_FLUSHBITS(&bitC);
}
FSE_flushCState(&bitC, &CState2);
FSE_flushCState(&bitC, &CState1);
return BIT_closeCStream(&bitC);
}
size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
const void* src, size_t srcSize,
const FSE_CTable* ct)
{
unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
if (fast)
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
else
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
}
size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
/* FSE_compress_wksp() :
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
* `wkspSize` size must be `(1<<tableLog)`.
*/
size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
{
BYTE* const ostart = (BYTE*) dst;
BYTE* op = ostart;
BYTE* const oend = ostart + dstSize;
U32 count[FSE_MAX_SYMBOL_VALUE+1];
S16 norm[FSE_MAX_SYMBOL_VALUE+1];
FSE_CTable* CTable = (FSE_CTable*)workSpace;
size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue);
void* scratchBuffer = (void*)(CTable + CTableSize);
size_t const scratchBufferSize = wkspSize - (CTableSize * sizeof(FSE_CTable));
/* init conditions */
if (wkspSize < FSE_WKSP_SIZE_U32(tableLog, maxSymbolValue)) return ERROR(tableLog_tooLarge);
if (srcSize <= 1) return 0; /* Not compressible */
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
/* Scan input and build symbol stats */
{ CHECK_V_F(maxCount, FSE_count(count, &maxSymbolValue, src, srcSize) );
if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
}
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) );
/* Write table description header */
{ CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
op += nc_err;
}
/* Compress */
CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) );
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) );
if (cSize == 0) return 0; /* not enough space for compressed data */
op += cSize;
}
/* check compressibility */
if ( (size_t)(op-ostart) >= srcSize-1 ) return 0;
return op-ostart;
}
typedef struct {
FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
BYTE scratchBuffer[1 << FSE_MAX_TABLELOG];
} fseWkspMax_t;
size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
{
fseWkspMax_t scratchBuffer;
FSE_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer));
}
size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
}
#endif /* FSE_COMMONDEFS_ONLY */

329
contrib/lizard/lib/entropy/fse_decompress.c Normal file
View File

@ -0,0 +1,329 @@
/* ******************************************************************
FSE : Finite State Entropy decoder
Copyright (C) 2013-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
/* **************************************************************
* Compiler specifics
****************************************************************/
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif
/* **************************************************************
* Includes
****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */
#include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */
#include "bitstream.h"
#define FSE_STATIC_LINKING_ONLY
#include "fse.h"
/* **************************************************************
* Error Management
****************************************************************/
#define FSE_isError ERR_isError
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/* check and forward error code */
#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
/* **************************************************************
* Templates
****************************************************************/
/*
designed to be included
for type-specific functions (template emulation in C)
Objective is to write these functions only once, for improved maintenance
*/
/* safety checks */
#ifndef FSE_FUNCTION_EXTENSION
# error "FSE_FUNCTION_EXTENSION must be defined"
#endif
#ifndef FSE_FUNCTION_TYPE
# error "FSE_FUNCTION_TYPE must be defined"
#endif
/* Function names */
#define FSE_CAT(X,Y) X##Y
#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
/* Function templates */
FSE_DTable* FSE_createDTable (unsigned tableLog)
{
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
}
void FSE_freeDTable (FSE_DTable* dt)
{
free(dt);
}
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
U32 const maxSV1 = maxSymbolValue + 1;
U32 const tableSize = 1 << tableLog;
U32 highThreshold = tableSize-1;
/* Sanity Checks */
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
/* Init, lay down lowprob symbols */
{ FSE_DTableHeader DTableH;
DTableH.tableLog = (U16)tableLog;
DTableH.fastMode = 1;
{ S16 const largeLimit= (S16)(1 << (tableLog-1));
U32 s;
for (s=0; s<maxSV1; s++) {
if (normalizedCounter[s]==-1) {
tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
symbolNext[s] = 1;
} else {
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
symbolNext[s] = normalizedCounter[s];
} } }
memcpy(dt, &DTableH, sizeof(DTableH));
}
/* Spread symbols */
{ U32 const tableMask = tableSize-1;
U32 const step = FSE_TABLESTEP(tableSize);
U32 s, position = 0;
for (s=0; s<maxSV1; s++) {
int i;
for (i=0; i<normalizedCounter[s]; i++) {
tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
position = (position + step) & tableMask;
while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
} }
if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
}
/* Build Decoding table */
{ U32 u;
for (u=0; u<tableSize; u++) {
FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
U16 nextState = symbolNext[symbol]++;
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
} }
return 0;
}
#ifndef FSE_COMMONDEFS_ONLY
/*-*******************************************************
* Decompression (Byte symbols)
*********************************************************/
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
{
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
void* dPtr = dt + 1;
FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
DTableH->tableLog = 0;
DTableH->fastMode = 0;
cell->newState = 0;
cell->symbol = symbolValue;
cell->nbBits = 0;
return 0;
}
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
{
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
void* dPtr = dt + 1;
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSV1 = tableMask+1;
unsigned s;
/* Sanity checks */
if (nbBits < 1) return ERROR(GENERIC); /* min size */
/* Build Decoding Table */
DTableH->tableLog = (U16)nbBits;
DTableH->fastMode = 1;
for (s=0; s<maxSV1; s++) {
dinfo[s].newState = 0;
dinfo[s].symbol = (BYTE)s;
dinfo[s].nbBits = (BYTE)nbBits;
}
return 0;
}
FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt, const unsigned fast)
{
BYTE* const ostart = (BYTE*) dst;
BYTE* op = ostart;
BYTE* const omax = op + maxDstSize;
BYTE* const olimit = omax-3;
BIT_DStream_t bitD;
FSE_DState_t state1;
FSE_DState_t state2;
/* Init */
CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
FSE_initDState(&state1, &bitD, dt);
FSE_initDState(&state2, &bitD, dt);
#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
/* 4 symbols per loop */
for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) {
op[0] = FSE_GETSYMBOL(&state1);
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
BIT_reloadDStream(&bitD);
op[1] = FSE_GETSYMBOL(&state2);
if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
{ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
op[2] = FSE_GETSYMBOL(&state1);
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
BIT_reloadDStream(&bitD);
op[3] = FSE_GETSYMBOL(&state2);
}
/* tail */
/* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
while (1) {
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
*op++ = FSE_GETSYMBOL(&state1);
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
*op++ = FSE_GETSYMBOL(&state2);
break;
}
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
*op++ = FSE_GETSYMBOL(&state2);
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
*op++ = FSE_GETSYMBOL(&state1);
break;
} }
return op-ostart;
}
size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt)
{
const void* ptr = dt;
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
const U32 fastMode = DTableH->fastMode;
/* select fast mode (static) */
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
}
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog)
{
const BYTE* const istart = (const BYTE*)cSrc;
const BYTE* ip = istart;
short counting[FSE_MAX_SYMBOL_VALUE+1];
unsigned tableLog;
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
/* normal FSE decoding mode */
size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
if (FSE_isError(NCountLength)) return NCountLength;
//if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */
if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
ip += NCountLength;
cSrcSize -= NCountLength;
CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) );
return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */
}
typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
{
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG);
}
#endif /* FSE_COMMONDEFS_ONLY */

250
contrib/lizard/lib/entropy/huf.h Normal file
View File

@ -0,0 +1,250 @@
/* ******************************************************************
Huffman coder, part of New Generation Entropy library
header file
Copyright (C) 2013-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
****************************************************************** */
#ifndef HUF_H_298734234
#define HUF_H_298734234
#if defined (__cplusplus)
extern "C" {
#endif
/* *** Dependencies *** */
#include <stddef.h> /* size_t */
/*-*** PUBLIC_API : control library symbols visibility *** */
#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
# define HUF_PUBLIC_API __attribute__ ((visibility ("default")))
#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
# define HUF_PUBLIC_API __declspec(dllexport)
#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
# define HUF_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define HUF_PUBLIC_API
#endif
/* *** simple functions *** */
/**
HUF_compress() :
Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
'dst' buffer must be already allocated.
Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
`srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
@return : size of compressed data (<= `dstCapacity`).
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single repeated byte symbol (RLE compression).
if HUF_isError(return), compression failed (more details using HUF_getErrorName())
*/
HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/**
HUF_decompress() :
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
into already allocated buffer 'dst', of minimum size 'dstSize'.
`originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
Note : in contrast with FSE, HUF_decompress can regenerate
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
because it knows size to regenerate.
@return : size of regenerated data (== originalSize),
or an error code, which can be tested using HUF_isError()
*/
HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize);
/* *** Tool functions *** */
#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
/* Error Management */
HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
/* *** Advanced function *** */
/** HUF_compress2() :
* Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` .
* `tableLog` must be `<= HUF_TABLELOG_MAX` . */
HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
/** HUF_compress4X_wksp() :
* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */
HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least 1024 unsigned */
#ifdef HUF_STATIC_LINKING_ONLY
/* *** Dependencies *** */
#include "mem.h" /* U32 */
/* *** Constants *** */
#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
#define HUF_SYMBOLVALUE_MAX 255
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
# error "HUF_TABLELOG_MAX is too large !"
#endif
/* ****************************************
* Static allocation
******************************************/
/* HUF buffer bounds */
#define HUF_CTABLEBOUND 129
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* static allocation of HUF's Compression Table */
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
U32 name##hb[maxSymbolValue+1]; \
void* name##hv = &(name##hb); \
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
/* static allocation of HUF's DTable */
typedef U32 HUF_DTable;
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
/* ****************************************
* Advanced decompression functions
******************************************/
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
/* ****************************************
* HUF detailed API
******************************************/
/*!
HUF_compress() does the following:
1. count symbol occurrence from source[] into table count[] using FSE_count()
2. (optional) refine tableLog using HUF_optimalTableLog()
3. build Huffman table from count using HUF_buildCTable()
4. save Huffman table to memory buffer using HUF_writeCTable()
5. encode the data stream using HUF_compress4X_usingCTable()
The following API allows targeting specific sub-functions for advanced tasks.
For example, it's possible to compress several blocks using the same 'CTable',
or to save and regenerate 'CTable' using external methods.
*/
/* FSE_count() : find it within "fse.h" */
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
/** HUF_buildCTable_wksp() :
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
*/
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize);
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
@return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize);
/** HUF_readCTable() :
* Loading a CTable saved with HUF_writeCTable() */
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
/*
HUF_decompress() does the following:
1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
2. build Huffman table from save, using HUF_readDTableXn()
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
*/
/** HUF_selectDecoder() :
* Tells which decoder is likely to decode faster,
* based on a set of pre-determined metrics.
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize);
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
/* single stream variants */
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least 1024 unsigned */
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#endif /* HUF_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
#endif /* HUF_H_298734234 */

612
contrib/lizard/lib/entropy/huf_compress.c Normal file
View File

@ -0,0 +1,612 @@
/* ******************************************************************
Huffman encoder, part of New Generation Entropy library
Copyright (C) 2013-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
/* **************************************************************
* Compiler specifics
****************************************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
/* **************************************************************
* Includes
****************************************************************/
#include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */
#include "bitstream.h"
#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */
#include "fse.h" /* header compression */
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
/* **************************************************************
* Error Management
****************************************************************/
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f
#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
/* **************************************************************
* Utils
****************************************************************/
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
{
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
}
/* *******************************************************
* HUF : Huffman block compression
*********************************************************/
/* HUF_compressWeights() :
* Same as FSE_compress(), but dedicated to huff0's weights compression.
* The use case needs much less stack memory.
* Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX.
*/
#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6
size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize)
{
BYTE* const ostart = (BYTE*) dst;
BYTE* op = ostart;
BYTE* const oend = ostart + dstSize;
U32 maxSymbolValue = HUF_TABLELOG_MAX;
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
U32 count[HUF_TABLELOG_MAX+1];
S16 norm[HUF_TABLELOG_MAX+1];
/* init conditions */
if (wtSize <= 1) return 0; /* Not compressible */
/* Scan input and build symbol stats */
{ CHECK_V_F(maxCount, FSE_count_simple(count, &maxSymbolValue, weightTable, wtSize) );
if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
}
tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue);
CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) );
/* Write table description header */
{ CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) );
op += hSize;
}
/* Compress */
CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) );
{ CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) );
if (cSize == 0) return 0; /* not enough space for compressed data */
op += cSize;
}
return op-ostart;
}
struct HUF_CElt_s {
U16 val;
BYTE nbBits;
}; /* typedef'd to HUF_CElt within "huf.h" */
/*! HUF_writeCTable() :
`CTable` : huffman tree to save, using huf representation.
@return : size of saved CTable */
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
{
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
BYTE* op = (BYTE*)dst;
U32 n;
/* check conditions */
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
/* convert to weight */
bitsToWeight[0] = 0;
for (n=1; n<huffLog+1; n++)
bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
for (n=0; n<maxSymbolValue; n++)
huffWeight[n] = bitsToWeight[CTable[n].nbBits];
/* attempt weights compression by FSE */
{ CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, huffWeight, maxSymbolValue) );
if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */
op[0] = (BYTE)hSize;
return hSize+1;
} }
/* write raw values as 4-bits (max : 15) */
if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */
for (n=0; n<maxSymbolValue; n+=2)
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
return ((maxSymbolValue+1)/2) + 1;
}
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
{
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
U32 tableLog = 0;
U32 nbSymbols = 0;
/* get symbol weights */
CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize));
/* check result */
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
/* Prepare base value per rank */
{ U32 n, nextRankStart = 0;
for (n=1; n<=tableLog; n++) {
U32 current = nextRankStart;
nextRankStart += (rankVal[n] << (n-1));
rankVal[n] = current;
} }
/* fill nbBits */
{ U32 n; for (n=0; n<nbSymbols; n++) {
const U32 w = huffWeight[n];
CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
} }
/* fill val */
{ U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */
U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
/* determine stating value per rank */
valPerRank[tableLog+1] = 0; /* for w==0 */
{ U16 min = 0;
U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */
valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n];
min >>= 1;
} }
/* assign value within rank, symbol order */
{ U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; }
}
return readSize;
}
typedef struct nodeElt_s {
U32 count;
U16 parent;
BYTE byte;
BYTE nbBits;
} nodeElt;
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
{
const U32 largestBits = huffNode[lastNonNull].nbBits;
if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */
/* there are several too large elements (at least >= 2) */
{ int totalCost = 0;
const U32 baseCost = 1 << (largestBits - maxNbBits);
U32 n = lastNonNull;
while (huffNode[n].nbBits > maxNbBits) {
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
huffNode[n].nbBits = (BYTE)maxNbBits;
n --;
} /* n stops at huffNode[n].nbBits <= maxNbBits */
while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */
/* renorm totalCost */
totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */
/* repay normalized cost */
{ U32 const noSymbol = 0xF0F0F0F0;
U32 rankLast[HUF_TABLELOG_MAX+2];
int pos;
/* Get pos of last (smallest) symbol per rank */
memset(rankLast, 0xF0, sizeof(rankLast));
{ U32 currentNbBits = maxNbBits;
for (pos=n ; pos >= 0; pos--) {
if (huffNode[pos].nbBits >= currentNbBits) continue;
currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */
rankLast[maxNbBits-currentNbBits] = pos;
} }
while (totalCost > 0) {
U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
U32 highPos = rankLast[nBitsToDecrease];
U32 lowPos = rankLast[nBitsToDecrease-1];
if (highPos == noSymbol) continue;
if (lowPos == noSymbol) break;
{ U32 const highTotal = huffNode[highPos].count;
U32 const lowTotal = 2 * huffNode[lowPos].count;
if (highTotal <= lowTotal) break;
} }
/* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
nBitsToDecrease ++;
totalCost -= 1 << (nBitsToDecrease-1);
if (rankLast[nBitsToDecrease-1] == noSymbol)
rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */
huffNode[rankLast[nBitsToDecrease]].nbBits ++;
if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
rankLast[nBitsToDecrease] = noSymbol;
else {
rankLast[nBitsToDecrease]--;
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
} } /* while (totalCost > 0) */
while (totalCost < 0) { /* Sometimes, cost correction overshoot */
if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */
while (huffNode[n].nbBits == maxNbBits) n--;
huffNode[n+1].nbBits--;
rankLast[1] = n+1;
totalCost++;
continue;
}
huffNode[ rankLast[1] + 1 ].nbBits--;
rankLast[1]++;
totalCost ++;
} } } /* there are several too large elements (at least >= 2) */
return maxNbBits;
}
typedef struct {
U32 base;
U32 current;
} rankPos;
static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
{
rankPos rank[32];
U32 n;
memset(rank, 0, sizeof(rank));
for (n=0; n<=maxSymbolValue; n++) {
U32 r = BIT_highbit32(count[n] + 1);
rank[r].base ++;
}
for (n=30; n>0; n--) rank[n-1].base += rank[n].base;
for (n=0; n<32; n++) rank[n].current = rank[n].base;
for (n=0; n<=maxSymbolValue; n++) {
U32 const c = count[n];
U32 const r = BIT_highbit32(c+1) + 1;
U32 pos = rank[r].current++;
while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--;
huffNode[pos].count = c;
huffNode[pos].byte = (BYTE)n;
}
}
/** HUF_buildCTable_wksp() :
* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned.
*/
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
typedef nodeElt huffNodeTable[2*HUF_SYMBOLVALUE_MAX+1 +1];
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
{
nodeElt* const huffNode0 = (nodeElt*)workSpace;
nodeElt* const huffNode = huffNode0+1;
U32 n, nonNullRank;
int lowS, lowN;
U16 nodeNb = STARTNODE;
U32 nodeRoot;
/* safety checks */
if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); /* workSpace is not large enough */
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
memset(huffNode0, 0, sizeof(huffNodeTable));
/* sort, decreasing order */
HUF_sort(huffNode, count, maxSymbolValue);
/* init for parents */
nonNullRank = maxSymbolValue;
while(huffNode[nonNullRank].count == 0) nonNullRank--;
lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
nodeNb++; lowS-=2;
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */
/* create parents */
while (nodeNb <= nodeRoot) {
U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
huffNode[n1].parent = huffNode[n2].parent = nodeNb;
nodeNb++;
}
/* distribute weights (unlimited tree height) */
huffNode[nodeRoot].nbBits = 0;
for (n=nodeRoot-1; n>=STARTNODE; n--)
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
for (n=0; n<=nonNullRank; n++)
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
/* enforce maxTableLog */
maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
/* fill result into tree (val, nbBits) */
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
for (n=0; n<=nonNullRank; n++)
nbPerRank[huffNode[n].nbBits]++;
/* determine stating value per rank */
{ U16 min = 0;
for (n=maxNbBits; n>0; n--) {
valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n];
min >>= 1;
} }
for (n=0; n<=maxSymbolValue; n++)
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
for (n=0; n<=maxSymbolValue; n++)
tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
}
return maxNbBits;
}
/** HUF_buildCTable() :
* Note : count is used before tree is written, so they can safely overlap
*/
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
{
huffNodeTable nodeTable;
return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
}
static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
{
BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
}
size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
#define HUF_FLUSHBITS_1(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream)
#define HUF_FLUSHBITS_2(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream)
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
{
const BYTE* ip = (const BYTE*) src;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = ostart + dstSize;
BYTE* op = ostart;
size_t n;
const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
BIT_CStream_t bitC;
/* init */
if (dstSize < 8) return 0; /* not enough space to compress */
{ size_t const initErr = BIT_initCStream(&bitC, op, oend-op);
if (HUF_isError(initErr)) return 0; }
n = srcSize & ~3; /* join to mod 4 */
switch (srcSize & 3)
{
case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable);
HUF_FLUSHBITS_2(&bitC);
// fallthrough
case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
HUF_FLUSHBITS_1(&bitC);
// fallthrough
case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
HUF_FLUSHBITS(&bitC);
// fallthrough
case 0 :
default: ;
}
for (; n>0; n-=4) { /* note : n&3==0 at this stage */
HUF_encodeSymbol(&bitC, ip[n- 1], CTable);
HUF_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n- 2], CTable);
HUF_FLUSHBITS_2(&bitC);
HUF_encodeSymbol(&bitC, ip[n- 3], CTable);
HUF_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n- 4], CTable);
HUF_FLUSHBITS(&bitC);
}
return BIT_closeCStream(&bitC);
}
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
{
size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */
const BYTE* ip = (const BYTE*) src;
const BYTE* const iend = ip + srcSize;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
BYTE* op = ostart;
if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */
if (srcSize < 12) return 0; /* no saving possible : too small input */
op += 6; /* jumpTable */
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
if (cSize==0) return 0;
MEM_writeLE16(ostart, (U16)cSize);
op += cSize;
}
ip += segmentSize;
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
if (cSize==0) return 0;
MEM_writeLE16(ostart+2, (U16)cSize);
op += cSize;
}
ip += segmentSize;
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) );
if (cSize==0) return 0;
MEM_writeLE16(ostart+4, (U16)cSize);
op += cSize;
}
ip += segmentSize;
{ CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) );
if (cSize==0) return 0;
op += cSize;
}
return op-ostart;
}
/* `workSpace` must a table of at least 1024 unsigned */
static size_t HUF_compress_internal (
void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
unsigned singleStream,
void* workSpace, size_t wkspSize)
{
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = ostart + dstSize;
BYTE* op = ostart;
union {
U32 count[HUF_SYMBOLVALUE_MAX+1];
HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
} table; /* `count` can overlap with `CTable`; saves 1 KB */
/* checks & inits */
if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC);
if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
if (!dstSize) return 0; /* cannot fit within dst budget */
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
/* Scan input and build symbol stats */
{ CHECK_V_F(largest, FSE_count_wksp (table.count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) );
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
}
/* Build Huffman Tree */
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
{ CHECK_V_F(maxBits, HUF_buildCTable_wksp (table.CTable, table.count, maxSymbolValue, huffLog, workSpace, wkspSize) );
huffLog = (U32)maxBits;
}
/* Write table description header */
{ CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, table.CTable, maxSymbolValue, huffLog) );
if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */
op += hSize;
}
/* Compress */
{ size_t const cSize = (singleStream) ?
HUF_compress1X_usingCTable(op, oend - op, src, srcSize, table.CTable) : /* single segment */
HUF_compress4X_usingCTable(op, oend - op, src, srcSize, table.CTable);
if (HUF_isError(cSize)) return cSize;
if (cSize==0) return 0; /* uncompressible */
op += cSize;
}
/* check compressibility */
if ((size_t)(op-ostart) >= srcSize-1)
return 0;
return op-ostart;
}
size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize)
{
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize);
}
size_t HUF_compress1X (void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog)
{
unsigned workSpace[1024];
return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
}
size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize)
{
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize);
}
size_t HUF_compress2 (void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog)
{
unsigned workSpace[1024];
return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace));
}
size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT);
}

885
contrib/lizard/lib/entropy/huf_decompress.c Normal file
View File

@ -0,0 +1,885 @@
/* ******************************************************************
Huffman decoder, part of New Generation Entropy library
Copyright (C) 2013-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
/* **************************************************************
* Compiler specifics
****************************************************************/
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
/* inline is defined */
#elif defined(_MSC_VER) || defined(__GNUC__)
# define inline __inline
#else
# define inline /* disable inline */
#endif
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
/* **************************************************************
* Dependencies
****************************************************************/
#include <string.h> /* memcpy, memset */
#include "bitstream.h" /* BIT_* */
#include "fse.h" /* header compression */
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
/* **************************************************************
* Error Management
****************************************************************/
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/*-***************************/
/* generic DTableDesc */
/*-***************************/
typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc;
static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
{
DTableDesc dtd;
memcpy(&dtd, table, sizeof(dtd));
return dtd;
}
/*-***************************/
/* single-symbol decoding */
/*-***************************/
typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
{
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
U32 tableLog = 0;
U32 nbSymbols = 0;
size_t iSize;
void* const dtPtr = DTable + 1;
HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize;
/* Table header */
{ DTableDesc dtd = HUF_getDTableDesc(DTable);
if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */
dtd.tableType = 0;
dtd.tableLog = (BYTE)tableLog;
memcpy(DTable, &dtd, sizeof(dtd));
}
/* Prepare ranks */
{ U32 n, nextRankStart = 0;
for (n=1; n<tableLog+1; n++) {
U32 current = nextRankStart;
nextRankStart += (rankVal[n] << (n-1));
rankVal[n] = current;
} }
/* fill DTable */
{ U32 n;
for (n=0; n<nbSymbols; n++) {
U32 const w = huffWeight[n];
U32 const length = (1 << w) >> 1;
U32 i;
HUF_DEltX2 D;
D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w);
for (i = rankVal[w]; i < rankVal[w] + length; i++)
dt[i] = D;
rankVal[w] += length;
} }
return iSize;
}
static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog)
{
size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
BYTE const c = dt[val].byte;
BIT_skipBits(Dstream, dt[val].nbBits);
return c;
}
#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \
*ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
if (MEM_64bits()) \
HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
{
BYTE* const pStart = p;
/* up to 4 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) {
HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
}
/* closer to the end */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd))
HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
/* no more data to retrieve from bitstream, hence no need to reload */
while (p < pEnd)
HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
return pEnd-pStart;
}
static size_t HUF_decompress1X2_usingDTable_internal(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + dstSize;
const void* dtPtr = DTable + 1;
const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
BIT_DStream_t bitD;
DTableDesc const dtd = HUF_getDTableDesc(DTable);
U32 const dtLog = dtd.tableLog;
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode)) return errorCode; }
HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog);
/* check */
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
return dstSize;
}
size_t HUF_decompress1X2_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 0) return ERROR(GENERIC);
return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
}
size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
const BYTE* ip = (const BYTE*) cSrc;
size_t const hSize = HUF_readDTableX2 (DCtx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize; cSrcSize -= hSize;
return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
}
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
return HUF_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize);
}
static size_t HUF_decompress4X2_usingDTable_internal(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
/* Check */
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{ const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable + 1;
const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
/* Init */
BIT_DStream_t bitD1;
BIT_DStream_t bitD2;
BIT_DStream_t bitD3;
BIT_DStream_t bitD4;
size_t const length1 = MEM_readLE16(istart);
size_t const length2 = MEM_readLE16(istart+2);
size_t const length3 = MEM_readLE16(istart+4);
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
DTableDesc const dtd = HUF_getDTableDesc(DTable);
U32 const dtLog = dtd.tableLog;
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode; }
/* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) {
HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
/* check */
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
}
size_t HUF_decompress4X2_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 0) return ERROR(GENERIC);
return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
}
size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
const BYTE* ip = (const BYTE*) cSrc;
size_t const hSize = HUF_readDTableX2 (dctx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize; cSrcSize -= hSize;
return HUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx);
}
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
}
/* *************************/
/* double-symbols decoding */
/* *************************/
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
/* HUF_fillDTableX4Level2() :
* `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed,
const U32* rankValOrigin, const int minWeight,
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize,
U32 nbBitsBaseline, U16 baseSeq)
{
HUF_DEltX4 DElt;
U32 rankVal[HUF_TABLELOG_MAX + 1];
/* get pre-calculated rankVal */
memcpy(rankVal, rankValOrigin, sizeof(rankVal));
/* fill skipped values */
if (minWeight>1) {
U32 i, skipSize = rankVal[minWeight];
MEM_writeLE16(&(DElt.sequence), baseSeq);
DElt.nbBits = (BYTE)(consumed);
DElt.length = 1;
for (i = 0; i < skipSize; i++)
DTable[i] = DElt;
}
/* fill DTable */
{ U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */
const U32 symbol = sortedSymbols[s].symbol;
const U32 weight = sortedSymbols[s].weight;
const U32 nbBits = nbBitsBaseline - weight;
const U32 length = 1 << (sizeLog-nbBits);
const U32 start = rankVal[weight];
U32 i = start;
const U32 end = start + length;
MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8)));
DElt.nbBits = (BYTE)(nbBits + consumed);
DElt.length = 2;
do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */
rankVal[weight] += length;
} }
}
typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1];
static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
const sortedSymbol_t* sortedList, const U32 sortedListSize,
const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
const U32 nbBitsBaseline)
{
U32 rankVal[HUF_TABLELOG_MAX + 1];
const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
const U32 minBits = nbBitsBaseline - maxWeight;
U32 s;
memcpy(rankVal, rankValOrigin, sizeof(rankVal));
/* fill DTable */
for (s=0; s<sortedListSize; s++) {
const U16 symbol = sortedList[s].symbol;
const U32 weight = sortedList[s].weight;
const U32 nbBits = nbBitsBaseline - weight;
const U32 start = rankVal[weight];
const U32 length = 1 << (targetLog-nbBits);
if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */
U32 sortedRank;
int minWeight = nbBits + scaleLog;
if (minWeight < 1) minWeight = 1;
sortedRank = rankStart[minWeight];
HUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits,
rankValOrigin[nbBits], minWeight,
sortedList+sortedRank, sortedListSize-sortedRank,
nbBitsBaseline, symbol);
} else {
HUF_DEltX4 DElt;
MEM_writeLE16(&(DElt.sequence), symbol);
DElt.nbBits = (BYTE)(nbBits);
DElt.length = 1;
{ U32 const end = start + length;
U32 u;
for (u = start; u < end; u++) DTable[u] = DElt;
} }
rankVal[weight] += length;
}
}
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
{
BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
U32 rankStats[HUF_TABLELOG_MAX + 1] = { 0 };
U32 rankStart0[HUF_TABLELOG_MAX + 2] = { 0 };
U32* const rankStart = rankStart0+1;
rankVal_t rankVal;
U32 tableLog, maxW, sizeOfSort, nbSymbols;
DTableDesc dtd = HUF_getDTableDesc(DTable);
U32 const maxTableLog = dtd.maxTableLog;
size_t iSize;
void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */
if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize;
/* check result */
if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
/* find maxWeight */
for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
/* Get start index of each weight */
{ U32 w, nextRankStart = 0;
for (w=1; w<maxW+1; w++) {
U32 current = nextRankStart;
nextRankStart += rankStats[w];
rankStart[w] = current;
}
rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
sizeOfSort = nextRankStart;
}
/* sort symbols by weight */
{ U32 s;
for (s=0; s<nbSymbols; s++) {
U32 const w = weightList[s];
U32 const r = rankStart[w]++;
sortedSymbol[r].symbol = (BYTE)s;
sortedSymbol[r].weight = (BYTE)w;
}
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
}
/* Build rankVal */
{ U32* const rankVal0 = rankVal[0];
{ int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */
U32 nextRankVal = 0;
U32 w;
for (w=1; w<maxW+1; w++) {
U32 current = nextRankVal;
nextRankVal += rankStats[w] << (w+rescale);
rankVal0[w] = current;
} }
{ U32 const minBits = tableLog+1 - maxW;
U32 consumed;
for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
U32* const rankValPtr = rankVal[consumed];
U32 w;
for (w = 1; w < maxW+1; w++) {
rankValPtr[w] = rankVal0[w] >> consumed;
} } } }
HUF_fillDTableX4(dt, maxTableLog,
sortedSymbol, sizeOfSort,
rankStart0, rankVal, maxW,
tableLog+1);
dtd.tableLog = (BYTE)maxTableLog;
dtd.tableType = 1;
memcpy(DTable, &dtd, sizeof(dtd));
return iSize;
}
static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog)
{
size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, dt+val, 2);
BIT_skipBits(DStream, dt[val].nbBits);
return dt[val].length;
}
static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog)
{
size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, dt+val, 1);
if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits);
else {
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
BIT_skipBits(DStream, dt[val].nbBits);
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
} }
return 1;
}
#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \
if (MEM_64bits()) \
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
{
BYTE* const pStart = p;
/* up to 8 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) {
HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
HUF_DECODE_SYMBOLX4_1(p, bitDPtr);
HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
}
/* closer to end : up to 2 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2))
HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
while (p <= pEnd-2)
HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
if (p < pEnd)
p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog);
return p-pStart;
}
static size_t HUF_decompress1X4_usingDTable_internal(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
BIT_DStream_t bitD;
/* Init */
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode)) return errorCode;
}
/* decode */
{ BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */
const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr;
DTableDesc const dtd = HUF_getDTableDesc(DTable);
HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog);
}
/* check */
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
size_t HUF_decompress1X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 1) return ERROR(GENERIC);
return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
}
size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
const BYTE* ip = (const BYTE*) cSrc;
size_t const hSize = HUF_readDTableX4 (DCtx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize; cSrcSize -= hSize;
return HUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
}
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX);
return HUF_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
}
static size_t HUF_decompress4X4_usingDTable_internal(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{ const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable+1;
const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr;
/* Init */
BIT_DStream_t bitD1;
BIT_DStream_t bitD2;
BIT_DStream_t bitD3;
BIT_DStream_t bitD4;
size_t const length1 = MEM_readLE16(istart);
size_t const length2 = MEM_readLE16(istart+2);
size_t const length3 = MEM_readLE16(istart+4);
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
size_t const segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
DTableDesc const dtd = HUF_getDTableDesc(DTable);
U32 const dtLog = dtd.tableLog;
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode; }
/* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) {
HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
HUF_DECODE_SYMBOLX4_2(op4, &bitD4);
HUF_DECODE_SYMBOLX4_1(op1, &bitD1);
HUF_DECODE_SYMBOLX4_1(op2, &bitD2);
HUF_DECODE_SYMBOLX4_1(op3, &bitD3);
HUF_DECODE_SYMBOLX4_1(op4, &bitD4);
HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
HUF_DECODE_SYMBOLX4_2(op4, &bitD4);
HUF_DECODE_SYMBOLX4_0(op1, &bitD1);
HUF_DECODE_SYMBOLX4_0(op2, &bitD2);
HUF_DECODE_SYMBOLX4_0(op3, &bitD3);
HUF_DECODE_SYMBOLX4_0(op4, &bitD4);
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 already verified within main loop */
/* finish bitStreams one by one */
HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog);
HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog);
HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog);
HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
/* check */
{ U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endCheck) return ERROR(corruption_detected); }
/* decoded size */
return dstSize;
}
}
size_t HUF_decompress4X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 1) return ERROR(GENERIC);
return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
}
size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
const BYTE* ip = (const BYTE*) cSrc;
size_t hSize = HUF_readDTableX4 (dctx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize; cSrcSize -= hSize;
return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
}
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX);
return HUF_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
}
/* ********************************/
/* Generic decompression selector */
/* ********************************/
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
DTableDesc const dtd = HUF_getDTableDesc(DTable);
return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) :
HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
}
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
DTableDesc const dtd = HUF_getDTableDesc(DTable);
return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) :
HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
}
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] =
{
/* single, double, quad */
{{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */
{{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */
{{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */
{{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */
{{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */
{{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */
{{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */
{{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */
{{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */
{{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */
{{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */
{{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */
{{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */
{{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */
{{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */
{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */
};
/** HUF_selectDecoder() :
* Tells which decoder is likely to decode faster,
* based on a set of pre-determined metrics.
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
{
/* decoder timing evaluation */
U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
U32 const D256 = (U32)(dstSize >> 8);
U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */
return DTime1 < DTime0;
}
typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
static const decompressionAlgo decompress[2] = { HUF_decompress4X2, HUF_decompress4X4 };
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
}
}
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
}
}
size_t HUF_decompress4X_hufOnly (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
}
}
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
}
}

372
contrib/lizard/lib/entropy/mem.h Normal file
View File

@ -0,0 +1,372 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
#ifndef MEM_H_MODULE
#define MEM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
#include <stddef.h> /* size_t, ptrdiff_t */
#include <string.h> /* memcpy */
/*-****************************************
* Compiler specifics
******************************************/
#if defined(_MSC_VER) /* Visual Studio */
# include <stdlib.h> /* _byteswap_ulong */
# include <intrin.h> /* _byteswap_* */
#endif
#if defined(__GNUC__)
# define MEM_STATIC static __inline __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define MEM_STATIC static inline
#elif defined(_MSC_VER)
# define MEM_STATIC static __inline
#else
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/* code only tested on 32 and 64 bits systems */
#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
/*-**************************************************************
* Basic Types
*****************************************************************/
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef int16_t S16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
typedef int64_t S64;
typedef intptr_t iPtrDiff;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef signed short S16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
typedef signed long long S64;
typedef ptrdiff_t iPtrDiff;
#endif
/*-**************************************************************
* Memory I/O
*****************************************************************/
/* MEM_FORCE_MEMORY_ACCESS :
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
* The below switch allow to select different access method for improved performance.
* Method 0 (default) : use `memcpy()`. Safe and portable.
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
* Method 2 : direct access. This method is portable but violate C standard.
* It can generate buggy code on targets depending on alignment.
* In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
* Prefer these methods in priority order (0 > 1 > 2)
*/
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define MEM_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define MEM_FORCE_MEMORY_ACCESS 1
# endif
#endif
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
MEM_STATIC unsigned MEM_isLittleEndian(void)
{
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
return one.c[0];
}
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
/* violates C standard, by lying on structure alignment.
Only use if no other choice to achieve best performance on target platform */
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
/* currently only defined for gcc and icc */
#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
__pragma( pack(push, 1) )
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign;
__pragma( pack(pop) )
#else
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
#endif
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
#else
/* default method, safe and standard.
can sometimes prove slower */
MEM_STATIC U16 MEM_read16(const void* memPtr)
{
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC U32 MEM_read32(const void* memPtr)
{
U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC U64 MEM_read64(const void* memPtr)
{
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC size_t MEM_readST(const void* memPtr)
{
size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC void MEM_write16(void* memPtr, U16 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
{
memcpy(memPtr, &value, sizeof(value));
}
#endif /* MEM_FORCE_MEMORY_ACCESS */
MEM_STATIC U32 MEM_swap32(U32 in)
{
#if defined(_MSC_VER) /* Visual Studio */
return _byteswap_ulong(in);
#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
return __builtin_bswap32(in);
#else
return ((in << 24) & 0xff000000 ) |
((in << 8) & 0x00ff0000 ) |
((in >> 8) & 0x0000ff00 ) |
((in >> 24) & 0x000000ff );
#endif
}
MEM_STATIC U64 MEM_swap64(U64 in)
{
#if defined(_MSC_VER) /* Visual Studio */
return _byteswap_uint64(in);
#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
return __builtin_bswap64(in);
#else
return ((in << 56) & 0xff00000000000000ULL) |
((in << 40) & 0x00ff000000000000ULL) |
((in << 24) & 0x0000ff0000000000ULL) |
((in << 8) & 0x000000ff00000000ULL) |
((in >> 8) & 0x00000000ff000000ULL) |
((in >> 24) & 0x0000000000ff0000ULL) |
((in >> 40) & 0x000000000000ff00ULL) |
((in >> 56) & 0x00000000000000ffULL);
#endif
}
MEM_STATIC size_t MEM_swapST(size_t in)
{
if (MEM_32bits())
return (size_t)MEM_swap32((U32)in);
else
return (size_t)MEM_swap64((U64)in);
}
/*=== Little endian r/w ===*/
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read16(memPtr);
else {
const BYTE* p = (const BYTE*)memPtr;
return (U16)(p[0] + (p[1]<<8));
}
}
MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
{
if (MEM_isLittleEndian()) {
MEM_write16(memPtr, val);
} else {
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val;
p[1] = (BYTE)(val>>8);
}
}
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
{
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
}
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
{
MEM_writeLE16(memPtr, (U16)val);
((BYTE*)memPtr)[2] = (BYTE)(val>>16);
}
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read32(memPtr);
else
return MEM_swap32(MEM_read32(memPtr));
}
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
MEM_write32(memPtr, val32);
else
MEM_write32(memPtr, MEM_swap32(val32));
}
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read64(memPtr);
else
return MEM_swap64(MEM_read64(memPtr));
}
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
MEM_write64(memPtr, val64);
else
MEM_write64(memPtr, MEM_swap64(val64));
}
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
{
if (MEM_32bits())
return (size_t)MEM_readLE32(memPtr);
else
return (size_t)MEM_readLE64(memPtr);
}
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeLE32(memPtr, (U32)val);
else
MEM_writeLE64(memPtr, (U64)val);
}
/*=== Big endian r/w ===*/
MEM_STATIC U32 MEM_readBE32(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_swap32(MEM_read32(memPtr));
else
return MEM_read32(memPtr);
}
MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
MEM_write32(memPtr, MEM_swap32(val32));
else
MEM_write32(memPtr, val32);
}
MEM_STATIC U64 MEM_readBE64(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_swap64(MEM_read64(memPtr));
else
return MEM_read64(memPtr);
}
MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
MEM_write64(memPtr, MEM_swap64(val64));
else
MEM_write64(memPtr, val64);
}
MEM_STATIC size_t MEM_readBEST(const void* memPtr)
{
if (MEM_32bits())
return (size_t)MEM_readBE32(memPtr);
else
return (size_t)MEM_readBE64(memPtr);
}
MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeBE32(memPtr, (U32)val);
else
MEM_writeBE64(memPtr, (U64)val);
}
/* function safe only for comparisons */
MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
{
switch (length)
{
default :
case 4 : return MEM_read32(memPtr);
case 3 : if (MEM_isLittleEndian())
return MEM_read32(memPtr)<<8;
else
return MEM_read32(memPtr)>>8;
}
}
#if defined (__cplusplus)
}
#endif
#endif /* MEM_H_MODULE */

15
contrib/lizard/lib/liblizard.pc.in Normal file
View File

@ -0,0 +1,15 @@
# Lizard - Fast LZ compression algorithm
# Copyright (C) 2011-2014, Yann Collet
# Copyright (C) 2016-2017, Przemyslaw Skibinski <inikep@gmail.com>
# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
prefix=@PREFIX@
libdir=@LIBDIR@
includedir=@INCLUDEDIR@
Name: lizard
Description: extremely fast lossless compression algorithm library
URL: http://github.com/inikep/lizard
Version: @VERSION@
Libs: -L@LIBDIR@ -llizard
Cflags: -I@INCLUDEDIR@

504
contrib/lizard/lib/lizard_common.h Normal file
View File

@ -0,0 +1,504 @@
/*
Lizard - Fast LZ compression algorithm
Copyright (C) 2011-2015, Yann Collet
Copyright (C) 2016-2017, Przemyslaw Skibinski <inikep@gmail.com>
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Lizard source repository : https://github.com/inikep/lizard
*/
#ifndef LIZARD_COMMON_H_2983
#define LIZARD_COMMON_H_2983
#if defined (__cplusplus)
extern "C" {
#endif
/*-************************************
* Memory routines
**************************************/
#include <stdlib.h> /* malloc, calloc, free */
#include <string.h> /* memset, memcpy */
#include <stdint.h> /* intptr_t */
#include "entropy/mem.h"
#include "lizard_compress.h" /* LIZARD_GCC_VERSION */
//#define LIZARD_USE_LOGS
#define LIZARD_LOG_COMPRESS(...) //printf(__VA_ARGS__)
#define LIZARD_LOG_DECOMPRESS(...) //printf(__VA_ARGS__)
#define LIZARD_LOG_COMPRESS_LZ4(...) //printf(__VA_ARGS__)
#define COMPLOG_CODEWORDS_LZ4(...) //printf(__VA_ARGS__)
#define LIZARD_LOG_DECOMPRESS_LZ4(...) //printf(__VA_ARGS__)
#define DECOMPLOG_CODEWORDS_LZ4(...) //printf(__VA_ARGS__)
#define LIZARD_LOG_COMPRESS_LIZv1(...) //printf(__VA_ARGS__)
#define COMPLOG_CODEWORDS_LIZv1(...) //printf(__VA_ARGS__)
#define LIZARD_LOG_DECOMPRESS_LIZv1(...) //printf(__VA_ARGS__)
#define DECOMPLOG_CODEWORDS_LIZv1(...) //printf(__VA_ARGS__)
/*-************************************
* Common Constants
**************************************/
#define MINMATCH 4
//#define USE_LZ4_ONLY
//#define LIZARD_USE_TEST
#define LIZARD_DICT_SIZE (1<<24)
#define WILDCOPYLENGTH 16
#define LASTLITERALS WILDCOPYLENGTH
#define MFLIMIT (WILDCOPYLENGTH+MINMATCH)
#define LIZARD_MAX_PRICE (1<<28)
#define LIZARD_INIT_LAST_OFFSET 0
#define LIZARD_MAX_16BIT_OFFSET (1<<16)
#define MM_LONGOFF 16
#define LIZARD_BLOCK_SIZE_PAD (LIZARD_BLOCK_SIZE+32)
#define LIZARD_COMPRESS_ADD_BUF (5*LIZARD_BLOCK_SIZE_PAD)
#ifndef LIZARD_NO_HUFFMAN
#define LIZARD_COMPRESS_ADD_HUF HUF_compressBound(LIZARD_BLOCK_SIZE_PAD)
#define LIZARD_HUF_BLOCK_SIZE LIZARD_BLOCK_SIZE
#else
#define LIZARD_COMPRESS_ADD_HUF 0
#define LIZARD_HUF_BLOCK_SIZE 1
#endif
/* LZ4 codewords */
#define ML_BITS_LZ4 4
#define ML_MASK_LZ4 ((1U<<ML_BITS_LZ4)-1)
#define RUN_BITS_LZ4 (8-ML_BITS_LZ4)
#define RUN_MASK_LZ4 ((1U<<RUN_BITS_LZ4)-1)
/* LIZv1 codewords */
#define ML_BITS_LIZv1 4
#define RUN_BITS_LIZv1 3
#define ML_RUN_BITS (ML_BITS_LIZv1 + RUN_BITS_LIZv1)
#define MAX_SHORT_LITLEN 7
#define MAX_SHORT_MATCHLEN 15
#define LIZARD_LAST_LONG_OFF 31
/* header byte */
#define LIZARD_FLAG_LITERALS 1
#define LIZARD_FLAG_FLAGS 2
#define LIZARD_FLAG_OFFSET16 4
#define LIZARD_FLAG_OFFSET24 8
#define LIZARD_FLAG_LEN 16
#define LIZARD_FLAG_UNCOMPRESSED 128
/* stream numbers */
#define LIZARD_STREAM_LITERALS 0
#define LIZARD_STREAM_FLAGS 1
#define LIZARD_STREAM_OFFSET16 2
#define LIZARD_STREAM_OFFSET24 3
#define LIZARD_STREAM_LEN 4
#define LIZARD_STREAM_UNCOMPRESSED 5
typedef enum { Lizard_parser_fastSmall, Lizard_parser_fast, Lizard_parser_fastBig, Lizard_parser_noChain, Lizard_parser_hashChain, Lizard_parser_priceFast, Lizard_parser_lowestPrice, Lizard_parser_optimalPrice, Lizard_parser_optimalPriceBT } Lizard_parser_type; /* from faster to stronger */
typedef enum { Lizard_coderwords_LZ4, Lizard_coderwords_LIZv1 } Lizard_decompress_type;
typedef struct
{
U32 windowLog; /* largest match distance : impact decompression buffer size */
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /* dispatch table : larger == more memory, faster*/
U32 hashLog3; /* dispatch table : larger == more memory, faster*/
U32 searchNum; /* nb of searches : larger == more compression, slower*/
U32 searchLength; /* size of matches : larger == faster decompression */
U32 minMatchLongOff; /* min match size with offsets >= 1<<16 */
U32 sufficientLength; /* used only by optimal parser: size of matches which is acceptable: larger == more compression, slower */
U32 fullSearch; /* used only by optimal parser: perform full search of matches: 1 == more compression, slower */
Lizard_parser_type parserType;
Lizard_decompress_type decompressType;
} Lizard_parameters;
struct Lizard_stream_s
{
const BYTE* end; /* next block here to continue on current prefix */
const BYTE* base; /* All index relative to this position */
const BYTE* dictBase; /* alternate base for extDict */
U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more dict */
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 allocatedMemory;
int compressionLevel;
Lizard_parameters params;
U32 hashTableSize;
U32 chainTableSize;
U32* chainTable;
U32* hashTable;
int last_off;
const BYTE* off24pos;
U32 huffType;
U32 comprStreamLen;
BYTE* huffBase;
BYTE* huffEnd;
BYTE* offset16Base;
BYTE* offset24Base;
BYTE* lenBase;
BYTE* literalsBase;
BYTE* flagsBase;
BYTE* offset16Ptr;
BYTE* offset24Ptr;
BYTE* lenPtr;
BYTE* literalsPtr;
BYTE* flagsPtr;
BYTE* offset16End;
BYTE* offset24End;
BYTE* lenEnd;
BYTE* literalsEnd;
BYTE* flagsEnd;
U32 flagFreq[256];
U32 litFreq[256];
U32 litSum, flagSum;
U32 litPriceSum, log2LitSum, log2FlagSum;
U32 cachedPrice;
U32 cachedLitLength;
const BYTE* cachedLiterals;
const BYTE* diffBase;
const BYTE* srcBase;
const BYTE* destBase;
};
struct Lizard_streamDecode_s {
const BYTE* externalDict;
size_t extDictSize;
const BYTE* prefixEnd;
size_t prefixSize;
};
struct Lizard_dstream_s
{
const BYTE* offset16Ptr;
const BYTE* offset24Ptr;
const BYTE* lenPtr;
const BYTE* literalsPtr;
const BYTE* flagsPtr;
const BYTE* offset16End;
const BYTE* offset24End;
const BYTE* lenEnd;
const BYTE* literalsEnd;
const BYTE* flagsEnd;
const BYTE* diffBase;
intptr_t last_off;
};
typedef struct Lizard_dstream_s Lizard_dstream_t;
/* *************************************
* HC Pre-defined compression levels
***************************************/
#define LIZARD_WINDOWLOG_LZ4 16
#define LIZARD_CHAINLOG_LZ4 LIZARD_WINDOWLOG_LZ4
#define LIZARD_HASHLOG_LZ4 18
#define LIZARD_HASHLOG_LZ4SM 12
#define LIZARD_WINDOWLOG_LIZv1 22
#define LIZARD_CHAINLOG_LIZv1 LIZARD_WINDOWLOG_LIZv1
#define LIZARD_HASHLOG_LIZv1 18
static const Lizard_parameters Lizard_defaultParameters[LIZARD_MAX_CLEVEL+1-LIZARD_MIN_CLEVEL] =
{
/* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */
{ LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4SM, 0, 0, 0, 0, 0, 0, Lizard_parser_fastSmall, Lizard_coderwords_LZ4 }, // level 10
{ LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_fast, Lizard_coderwords_LZ4 }, // level 11
{ LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_noChain, Lizard_coderwords_LZ4 }, // level 12
{ LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 2, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 13
{ LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 4, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 14
{ LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 8, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 15
{ LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 16, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 16
{ LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 256, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 17
{ LIZARD_WINDOWLOG_LZ4, LIZARD_WINDOWLOG_LZ4+1, LIZARD_HASHLOG_LZ4, 16, 16, 4, 0, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LZ4 }, // level 18
{ LIZARD_WINDOWLOG_LZ4, LIZARD_WINDOWLOG_LZ4+1, 23, 16, 256, 4, 0, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LZ4 }, // level 19
/* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */
{ LIZARD_WINDOWLOG_LIZv1, 0, 14, 0, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_fastBig, Lizard_coderwords_LIZv1 }, // level 20
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 14, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 21
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 22
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 23
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 2, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 24
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 8, 4, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 25
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 8, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 26
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 128, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 27
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 28
{ 24, 25, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 29
#ifndef LIZARD_NO_HUFFMAN
/* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */
{ LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4SM, 0, 0, 0, 0, 0, 0, Lizard_parser_fastSmall, Lizard_coderwords_LZ4 }, // level 30
{ LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_fast, Lizard_coderwords_LZ4 }, // level 31
{ LIZARD_WINDOWLOG_LZ4, 0, 14, 0, 0, 0, 0, 0, 0, Lizard_parser_noChain, Lizard_coderwords_LZ4 }, // level 32
{ LIZARD_WINDOWLOG_LZ4, 0, LIZARD_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, Lizard_parser_noChain, Lizard_coderwords_LZ4 }, // level 33
{ LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 2, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 34
{ LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 4, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 35
{ LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 8, 5, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 36
{ LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 16, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 37
{ LIZARD_WINDOWLOG_LZ4, LIZARD_CHAINLOG_LZ4, LIZARD_HASHLOG_LZ4, 0, 256, 4, 0, 0, 0, Lizard_parser_hashChain, Lizard_coderwords_LZ4 }, // level 38
{ LIZARD_WINDOWLOG_LZ4, LIZARD_WINDOWLOG_LZ4+1, 23, 16, 256, 4, 0, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LZ4 }, // level 39
/* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */
{ LIZARD_WINDOWLOG_LIZv1, 0, 14, 0, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_fastBig, Lizard_coderwords_LIZv1 }, // level 40
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 14, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 41
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 0, 0, Lizard_parser_priceFast, Lizard_coderwords_LIZv1 }, // level 42
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, LIZARD_HASHLOG_LIZv1, 13, 1, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 43
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 2, 5, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 44
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 8, 4, MM_LONGOFF, 64, 0, Lizard_parser_lowestPrice, Lizard_coderwords_LIZv1 }, // level 45
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1, 23, 16, 8, 4, MM_LONGOFF, 64, 0, Lizard_parser_optimalPrice, Lizard_coderwords_LIZv1 }, // level 46
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 8, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 47
{ LIZARD_WINDOWLOG_LIZv1, LIZARD_CHAINLOG_LIZv1+1, 23, 16, 128, 4, MM_LONGOFF, 64, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 48
{ 24, 25, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, Lizard_parser_optimalPriceBT, Lizard_coderwords_LIZv1 }, // level 49
#endif
// { 10, 10, 10, 0, 0, 4, 0, 0, 0, Lizard_fast }, // min values
// { 24, 24, 28, 24, 1<<24, 7, 0, 1<<24, 2, Lizard_optimal_price }, // max values
};
/*-************************************
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h>
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */
#else
# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# if defined(__GNUC__) || defined(__clang__)
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif /* _MSC_VER */
#define LIZARD_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#if (LIZARD_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
# define expect(expr,value) (__builtin_expect ((expr),(value)) )
#else
# define expect(expr,value) (expr)
#endif
#define likely(expr) expect((expr) != 0, 1)
#define unlikely(expr) expect((expr) != 0, 0)
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
#define ALLOCATOR(n,s) calloc(n,s)
#define FREEMEM free
#define MEM_INIT memset
#ifndef MAX
#define MAX(a,b) ((a)>(b))?(a):(b)
#endif
#ifndef MIN
#define MIN(a,b) ((a)<(b)?(a):(b))
#endif
#if MINMATCH == 3
#define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr)<<8)
#else
#define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr))
#endif
/*-************************************
* Reading and writing into memory
**************************************/
#define STEPSIZE sizeof(size_t)
MEM_STATIC void Lizard_copy8(void* dst, const void* src)
{
memcpy(dst,src,8);
}
/* customized variant of memcpy, which can overwrite up to 7 bytes beyond dstEnd */
MEM_STATIC void Lizard_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
{
BYTE* d = (BYTE*)dstPtr;
const BYTE* s = (const BYTE*)srcPtr;
BYTE* const e = (BYTE*)dstEnd;
#if 0
const size_t l2 = 8 - (((size_t)d) & (sizeof(void*)-1));
Lizard_copy8(d,s); if (d>e-9) return;
d+=l2; s+=l2;
#endif /* join to align */
do { Lizard_copy8(d,s); d+=8; s+=8; } while (d<e);
}
MEM_STATIC void Lizard_wildCopy16(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd)
{
do {
Lizard_copy8(dstPtr, srcPtr);
Lizard_copy8(dstPtr+8, srcPtr+8);
dstPtr += 16;
srcPtr += 16;
}
while (dstPtr < dstEnd);
}
/*
* LIZARD_FORCE_SW_BITCOUNT
* Define this parameter if your target system or compiler does not support hardware bit count
*/
#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */
# define LIZARD_FORCE_SW_BITCOUNT
#endif
/* **************************************
* Function body to include for inlining
****************************************/
MEM_STATIC U32 Lizard_highbit32(U32 val)
{
# if defined(_MSC_VER) /* Visual */
unsigned long r=0;
_BitScanReverse(&r, val);
return (unsigned)r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
return 31 - __builtin_clz(val);
# else /* Software version */
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
int r;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
return r;
# endif
}
/*-************************************
* Common functions
**************************************/
MEM_STATIC unsigned Lizard_NbCommonBytes (register size_t val)
{
if (MEM_isLittleEndian()) {
if (MEM_64bits()) {
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LIZARD_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
} else /* 32 bits */ {
# if defined(_MSC_VER) && !defined(LIZARD_FORCE_SW_BITCOUNT)
unsigned long r;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
} else /* Big Endian CPU */ {
if (MEM_64bits()) {
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LIZARD_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT)
return (__builtin_clzll((U64)val) >> 3);
# else
unsigned r;
if (!(val>>32)) { r=4; } else { r=0; val>>=32; }
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
r += (!val);
return r;
# endif
} else /* 32 bits */ {
# if defined(_MSC_VER) && !defined(LIZARD_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif (defined(__clang__) || (LIZARD_GCC_VERSION >= 304)) && !defined(LIZARD_FORCE_SW_BITCOUNT)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
r += (!val);
return r;
# endif
}
}
}
MEM_STATIC unsigned Lizard_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
while (likely(pIn<pInLimit-(STEPSIZE-1))) {
size_t diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; }
pIn += Lizard_NbCommonBytes(diff);
return (unsigned)(pIn - pStart);
}
if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (unsigned)(pIn - pStart);
}
/* alias to functions with compressionLevel=1 */
int Lizard_sizeofState_MinLevel(void);
int Lizard_compress_MinLevel(const char* source, char* dest, int sourceSize, int maxDestSize);
int Lizard_compress_extState_MinLevel (void* state, const char* source, char* dest, int inputSize, int maxDestSize);
Lizard_stream_t* Lizard_resetStream_MinLevel (Lizard_stream_t* streamPtr);
Lizard_stream_t* Lizard_createStream_MinLevel(void);
#if defined (__cplusplus)
}
#endif
#endif /* LIZARD_COMMON_H_2983827168210 */

630
contrib/lizard/lib/lizard_compress.c Normal file
View File

@ -0,0 +1,630 @@
/*
Lizard - Fast LZ compression algorithm
Copyright (C) 2011-2015, Yann Collet
Copyright (C) 2016-2017, Przemyslaw Skibinski <inikep@gmail.com>
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOTLizard_hash4Ptr
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Lizard source repository : https://github.com/inikep/lizard
*/
/* *************************************
* Includes
***************************************/
#include "lizard_compress.h"
#include "lizard_common.h"
#include <stdio.h>
#include <stdint.h> // intptr_t
#ifndef USE_LZ4_ONLY
#ifdef LIZARD_USE_TEST
#include "test/lizard_common_test.h"
#include "test/lizard_compress_test.h"
#else
#include "lizard_compress_liz.h"
#endif
#endif
#include "lizard_compress_lz4.h"
#include "entropy/huf.h"
/* *************************************
* Local Macros
***************************************/
#define DELTANEXT(p) chainTable[(p) & contentMask]
#define LIZARD_MINIMAL_HUFF_GAIN(comprSize) (comprSize + (comprSize/8) + 512)
#define LIZARD_MINIMAL_BLOCK_GAIN(comprSize) (comprSize + (comprSize/32) + 512)
/*-************************************
* Local Utils
**************************************/
int Lizard_versionNumber (void) { return LIZARD_VERSION_NUMBER; }
int Lizard_compressBound(int isize) { return LIZARD_COMPRESSBOUND(isize); }
int Lizard_sizeofState_MinLevel() { return Lizard_sizeofState(LIZARD_MIN_CLEVEL); }
/* *************************************
* Hash functions
***************************************/
#define HASH_UPDATE_LIMIT 8 /* equal to MEM_read64 */
static const U32 prime4bytes = 2654435761U;
static const U64 prime5bytes = 889523592379ULL;
static const U64 prime6bytes = 227718039650203ULL;
static const U64 prime7bytes = 58295818150454627ULL;
#if MINMATCH == 3
static const U32 prime3bytes = 506832829U;
static U32 Lizard_hash3(U32 u, U32 h) { return (u * prime3bytes) << (32-24) >> (32-h) ; }
static size_t Lizard_hash3Ptr(const void* ptr, U32 h) { return Lizard_hash3(MEM_read32(ptr), h); }
#endif
static U32 Lizard_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
static size_t Lizard_hash4Ptr(const void* ptr, U32 h) { return Lizard_hash4(MEM_read32(ptr), h); }
static size_t Lizard_hash5(U64 u, U32 h) { return (size_t)((u * prime5bytes) << (64-40) >> (64-h)) ; }
static size_t Lizard_hash5Ptr(const void* p, U32 h) { return Lizard_hash5(MEM_read64(p), h); }
static size_t Lizard_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; }
static size_t Lizard_hash6Ptr(const void* p, U32 h) { return Lizard_hash6(MEM_read64(p), h); }
static size_t Lizard_hash7(U64 u, U32 h) { return (size_t)((u * prime7bytes) << (64-56) >> (64-h)) ; }
static size_t Lizard_hash7Ptr(const void* p, U32 h) { return Lizard_hash7(MEM_read64(p), h); }
static size_t Lizard_hashPtr(const void* p, U32 hBits, U32 mls)
{
switch(mls)
{
default:
case 4: return Lizard_hash4Ptr(p, hBits);
case 5: return Lizard_hash5Ptr(p, hBits);
case 6: return Lizard_hash6Ptr(p, hBits);
case 7: return Lizard_hash7Ptr(p, hBits);
}
}
/**************************************
* Internal functions
**************************************/
/** Lizard_count_2segments() :
* can count match length with `ip` & `match` in 2 different segments.
* convention : on reaching mEnd, match count continue starting from iStart
*/
static size_t Lizard_count_2segments(const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart)
{
const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd);
size_t const matchLength = Lizard_count(ip, match, vEnd);
if (match + matchLength != mEnd) return matchLength;
return matchLength + Lizard_count(ip+matchLength, iStart, iEnd);
}
void Lizard_initBlock(Lizard_stream_t* ctx)
{
ctx->offset16Ptr = ctx->offset16Base;
ctx->offset24Ptr = ctx->offset24Base;
ctx->lenPtr = ctx->lenBase;
ctx->literalsPtr = ctx->literalsBase;
ctx->flagsPtr = ctx->flagsBase;
ctx->last_off = LIZARD_INIT_LAST_OFFSET; /* reset last offset */
}
FORCE_INLINE int Lizard_writeStream(int useHuff, Lizard_stream_t* ctx, BYTE* streamPtr, uint32_t streamLen, BYTE** op, BYTE* oend)
{
if (useHuff && streamLen > 1024) {
#ifndef LIZARD_NO_HUFFMAN
int useHuffBuf;
if (*op + 6 > oend) { LIZARD_LOG_COMPRESS("*op[%p] + 6 > oend[%p]\n", *op, oend); return -1; }
useHuffBuf = ((size_t)(oend - (*op + 6)) < HUF_compressBound(streamLen));
if (useHuffBuf) {
if (streamLen > LIZARD_BLOCK_SIZE) { LIZARD_LOG_COMPRESS("streamLen[%d] > LIZARD_BLOCK_SIZE\n", streamLen); return -1; }
ctx->comprStreamLen = (U32)HUF_compress(ctx->huffBase, ctx->huffEnd - ctx->huffBase, streamPtr, streamLen);
} else {
ctx->comprStreamLen = (U32)HUF_compress(*op + 6, oend - (*op + 6), streamPtr, streamLen);
}
if (!HUF_isError(ctx->comprStreamLen)) {
if (ctx->comprStreamLen > 0 && (LIZARD_MINIMAL_HUFF_GAIN(ctx->comprStreamLen) < streamLen)) { /* compressible */
MEM_writeLE24(*op, streamLen);
MEM_writeLE24(*op+3, ctx->comprStreamLen);
if (useHuffBuf) {
if ((size_t)(oend - (*op + 6)) < ctx->comprStreamLen) { LIZARD_LOG_COMPRESS("*op[%p] oend[%p] comprStreamLen[%d]\n", *op, oend, (int)ctx->comprStreamLen); return -1; }
memcpy(*op + 6, ctx->huffBase, ctx->comprStreamLen);
}
*op += ctx->comprStreamLen + 6;
LIZARD_LOG_COMPRESS("HUF_compress streamLen=%d comprStreamLen=%d\n", (int)streamLen, (int)ctx->comprStreamLen);
return 1;
} else { LIZARD_LOG_COMPRESS("HUF_compress ERROR comprStreamLen=%d streamLen=%d\n", (int)ctx->comprStreamLen, (int)streamLen); }
} else { LIZARD_LOG_COMPRESS("HUF_compress ERROR %d: %s\n", (int)ctx->comprStreamLen, HUF_getErrorName(ctx->comprStreamLen)); }
#else
LIZARD_LOG_COMPRESS("compiled with LIZARD_NO_HUFFMAN\n");
(void)ctx;
return -1;
#endif
} else ctx->comprStreamLen = 0;
if (*op + 3 + streamLen > oend) { LIZARD_LOG_COMPRESS("*op[%p] + 3 + streamLen[%d] > oend[%p]\n", *op, streamLen, oend); return -1; }
MEM_writeLE24(*op, streamLen);
*op += 3;
memcpy(*op, streamPtr, streamLen);
*op += streamLen;
LIZARD_LOG_COMPRESS("Uncompressed streamLen=%d\n", (int)streamLen);
return 0;
}
int Lizard_writeBlock(Lizard_stream_t* ctx, const BYTE* ip, uint32_t inputSize, BYTE** op, BYTE* oend)
{
int res;
uint32_t flagsLen = (uint32_t)(ctx->flagsPtr - ctx->flagsBase);
uint32_t literalsLen = (uint32_t)(ctx->literalsPtr - ctx->literalsBase);
uint32_t lenLen = (uint32_t)(ctx->lenPtr - ctx->lenBase);
uint32_t offset16Len = (uint32_t)(ctx->offset16Ptr - ctx->offset16Base);
uint32_t offset24Len = (uint32_t)(ctx->offset24Ptr - ctx->offset24Base);
uint32_t sum = flagsLen + literalsLen + lenLen + offset16Len + offset24Len;
#ifdef LIZARD_USE_LOGS
uint32_t comprFlagsLen, comprLiteralsLen;
#endif
BYTE* start = *op;
if ((literalsLen < WILDCOPYLENGTH) || (sum+5*3+1 > inputSize)) goto _write_uncompressed;
*start = 0;
*op += 1;
res = Lizard_writeStream(0, ctx, ctx->lenBase, lenLen, op, oend);
if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_LEN);
res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_OFFSET16, ctx, ctx->offset16Base, offset16Len, op, oend);
if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_OFFSET16);
res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_OFFSET24, ctx, ctx->offset24Base, offset24Len, op, oend);
if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_OFFSET24);
res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_FLAGS, ctx, ctx->flagsBase, flagsLen, op, oend);
if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_FLAGS);
#ifdef LIZARD_USE_LOGS
comprFlagsLen = ctx->comprStreamLen;
#endif
res = Lizard_writeStream(ctx->huffType&LIZARD_FLAG_LITERALS, ctx, ctx->literalsBase, literalsLen, op, oend);
if (res < 0) goto _output_error; else *start += (BYTE)(res*LIZARD_FLAG_LITERALS);
#ifdef LIZARD_USE_LOGS
comprLiteralsLen = ctx->comprStreamLen;
sum = (int)(*op-start);
#endif
if (LIZARD_MINIMAL_BLOCK_GAIN((uint32_t)(*op-start)) > inputSize) goto _write_uncompressed;
LIZARD_LOG_COMPRESS("%d: total=%d block=%d flagsLen[%.2f%%]=%d comprFlagsLen[%.2f%%]=%d literalsLen[%.2f%%]=%d comprLiteralsLen[%.2f%%]=%d lenLen=%d offset16Len[%.2f%%]=%d offset24Len[%.2f%%]=%d\n", (int)(ip - ctx->srcBase),
(int)(*op - ctx->destBase), sum, (flagsLen*100.0)/sum, flagsLen, (comprFlagsLen*100.0)/sum, comprFlagsLen, (literalsLen*100.0)/sum, literalsLen, (comprLiteralsLen*100.0)/sum, comprLiteralsLen,
lenLen, (offset16Len*100.0)/sum, offset16Len, (offset24Len*100.0)/sum, offset24Len);
return 0;
_write_uncompressed:
LIZARD_LOG_COMPRESS("%d: total=%d block=%d UNCOMPRESSED inputSize=%u outSize=%d\n", (int)(ip - ctx->srcBase),
(int)(*op - ctx->destBase), (int)(*op-start), inputSize, (int)(oend-start));
if ((uint32_t)(oend - start) < inputSize + 4) goto _output_error;
*start = LIZARD_FLAG_UNCOMPRESSED;
*op = start + 1;
MEM_writeLE24(*op, inputSize);
*op += 3;
memcpy(*op, ip, inputSize);
*op += inputSize;
return 0;
_output_error:
LIZARD_LOG_COMPRESS("Lizard_writeBlock ERROR size=%d/%d flagsLen=%d literalsLen=%d lenLen=%d offset16Len=%d offset24Len=%d\n", (int)(*op-start), (int)(oend-start), flagsLen, literalsLen, lenLen, offset16Len, offset24Len);
return 1;
}
FORCE_INLINE int Lizard_encodeSequence (
Lizard_stream_t* ctx,
const BYTE** ip,
const BYTE** anchor,
size_t matchLength,
const BYTE* const match)
{
#ifdef USE_LZ4_ONLY
return Lizard_encodeSequence_LZ4(ctx, ip, anchor, matchLength, match);
#else
if (ctx->params.decompressType == Lizard_coderwords_LZ4)
return Lizard_encodeSequence_LZ4(ctx, ip, anchor, matchLength, match);
return Lizard_encodeSequence_LIZv1(ctx, ip, anchor, matchLength, match);
#endif
}
FORCE_INLINE int Lizard_encodeLastLiterals (
Lizard_stream_t* ctx,
const BYTE** ip,
const BYTE** anchor)
{
LIZARD_LOG_COMPRESS("Lizard_encodeLastLiterals Lizard_coderwords_LZ4=%d\n", ctx->params.decompressType == Lizard_coderwords_LZ4);
#ifdef USE_LZ4_ONLY
return Lizard_encodeLastLiterals_LZ4(ctx, ip, anchor);
#else
if (ctx->params.decompressType == Lizard_coderwords_LZ4)
return Lizard_encodeLastLiterals_LZ4(ctx, ip, anchor);
return Lizard_encodeLastLiterals_LIZv1(ctx, ip, anchor);
#endif
}
/**************************************
* Include parsers
**************************************/
#include "lizard_parser_hashchain.h"
#include "lizard_parser_nochain.h"
#include "lizard_parser_fast.h"
#include "lizard_parser_fastsmall.h"
#include "lizard_parser_fastbig.h"
#ifndef USE_LZ4_ONLY
#include "lizard_parser_optimal.h"
#include "lizard_parser_lowestprice.h"
#include "lizard_parser_pricefast.h"
#endif
int Lizard_verifyCompressionLevel(int compressionLevel)
{
if (compressionLevel > LIZARD_MAX_CLEVEL) compressionLevel = LIZARD_MAX_CLEVEL;
if (compressionLevel < LIZARD_MIN_CLEVEL) compressionLevel = LIZARD_DEFAULT_CLEVEL;
return compressionLevel;
}
int Lizard_sizeofState(int compressionLevel)
{
Lizard_parameters params;
U32 hashTableSize, chainTableSize;
compressionLevel = Lizard_verifyCompressionLevel(compressionLevel);
params = Lizard_defaultParameters[compressionLevel - LIZARD_MIN_CLEVEL];
// hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog3)+((size_t)1 << params.hashLog)));
hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog)));
chainTableSize = (U32)(sizeof(U32)*((size_t)1 << params.contentLog));
return sizeof(Lizard_stream_t) + hashTableSize + chainTableSize + LIZARD_COMPRESS_ADD_BUF + (int)LIZARD_COMPRESS_ADD_HUF;
}
static void Lizard_init(Lizard_stream_t* ctx, const BYTE* start)
{
// No need to use memset() on tables as values are always bound checked
#ifdef LIZARD_RESET_MEM
MEM_INIT((void*)ctx->hashTable, 0, ctx->hashTableSize);
MEM_INIT(ctx->chainTable, 0x01, ctx->chainTableSize);
#endif
// printf("memset hashTable=%p hashEnd=%p chainTable=%p chainEnd=%p\n", ctx->hashTable, ((BYTE*)ctx->hashTable) + ctx->hashTableSize, ctx->chainTable, ((BYTE*)ctx->chainTable)+ctx->chainTableSize);
ctx->nextToUpdate = LIZARD_DICT_SIZE;
ctx->base = start - LIZARD_DICT_SIZE;
ctx->end = start;
ctx->dictBase = start - LIZARD_DICT_SIZE;
ctx->dictLimit = LIZARD_DICT_SIZE;
ctx->lowLimit = LIZARD_DICT_SIZE;
ctx->last_off = LIZARD_INIT_LAST_OFFSET;
ctx->litSum = 0;
}
/* if ctx==NULL memory is allocated and returned as value */
Lizard_stream_t* Lizard_initStream(Lizard_stream_t* ctx, int compressionLevel)
{
Lizard_parameters params;
U32 hashTableSize, chainTableSize;
void *tempPtr;
compressionLevel = Lizard_verifyCompressionLevel(compressionLevel);
params = Lizard_defaultParameters[compressionLevel - LIZARD_MIN_CLEVEL];
// hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog3)+((size_t)1 << params.hashLog)));
hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog)));
chainTableSize = (U32)(sizeof(U32)*((size_t)1 << params.contentLog));
if (!ctx)
{
ctx = (Lizard_stream_t*)malloc(sizeof(Lizard_stream_t) + hashTableSize + chainTableSize + LIZARD_COMPRESS_ADD_BUF + LIZARD_COMPRESS_ADD_HUF);
if (!ctx) { printf("ERROR: Cannot allocate %d MB (compressionLevel=%d)\n", (int)(sizeof(Lizard_stream_t) + hashTableSize + chainTableSize)>>20, compressionLevel); return 0; }
LIZARD_LOG_COMPRESS("Allocated %d MB (compressionLevel=%d)\n", (int)(sizeof(Lizard_stream_t) + hashTableSize + chainTableSize)>>20, compressionLevel);
ctx->allocatedMemory = sizeof(Lizard_stream_t) + hashTableSize + chainTableSize + LIZARD_COMPRESS_ADD_BUF + (U32)LIZARD_COMPRESS_ADD_HUF;
// printf("malloc from=%p to=%p hashTable=%p hashEnd=%p chainTable=%p chainEnd=%p\n", ctx, ((BYTE*)ctx)+sizeof(Lizard_stream_t) + hashTableSize + chainTableSize, ctx->hashTable, ((BYTE*)ctx->hashTable) + hashTableSize, ctx->chainTable, ((BYTE*)ctx->chainTable)+chainTableSize);
}
tempPtr = ctx;
ctx->hashTable = (U32*)(tempPtr) + sizeof(Lizard_stream_t)/4;
ctx->hashTableSize = hashTableSize;
ctx->chainTable = ctx->hashTable + hashTableSize/4;
ctx->chainTableSize = chainTableSize;
ctx->params = params;
ctx->compressionLevel = (unsigned)compressionLevel;
if (compressionLevel < 30)
ctx->huffType = 0;
else
ctx->huffType = LIZARD_FLAG_LITERALS + LIZARD_FLAG_FLAGS; // + LIZARD_FLAG_OFFSET16 + LIZARD_FLAG_OFFSET24;
ctx->literalsBase = (BYTE*)ctx->hashTable + ctx->hashTableSize + ctx->chainTableSize;
ctx->flagsBase = ctx->literalsEnd = ctx->literalsBase + LIZARD_BLOCK_SIZE_PAD;
ctx->lenBase = ctx->flagsEnd = ctx->flagsBase + LIZARD_BLOCK_SIZE_PAD;
ctx->offset16Base = ctx->lenEnd = ctx->lenBase + LIZARD_BLOCK_SIZE_PAD;
ctx->offset24Base = ctx->offset16End = ctx->offset16Base + LIZARD_BLOCK_SIZE_PAD;
ctx->huffBase = ctx->offset24End = ctx->offset24Base + LIZARD_BLOCK_SIZE_PAD;
ctx->huffEnd = ctx->huffBase + LIZARD_COMPRESS_ADD_HUF;
return ctx;
}
Lizard_stream_t* Lizard_createStream(int compressionLevel)
{
Lizard_stream_t* ctx = Lizard_initStream(NULL, compressionLevel);
if (ctx) ctx->base = NULL;
return ctx;
}
/* initialization */
Lizard_stream_t* Lizard_resetStream(Lizard_stream_t* ctx, int compressionLevel)
{
size_t wanted = Lizard_sizeofState(compressionLevel);
if (ctx->allocatedMemory < wanted) {
Lizard_freeStream(ctx);
ctx = Lizard_createStream(compressionLevel);
} else {
Lizard_initStream(ctx, compressionLevel);
}
if (ctx) ctx->base = NULL;
return ctx;
}
int Lizard_freeStream(Lizard_stream_t* ctx)
{
if (ctx) {
free(ctx);
}
return 0;
}
int Lizard_loadDict(Lizard_stream_t* Lizard_streamPtr, const char* dictionary, int dictSize)
{
Lizard_stream_t* ctxPtr = (Lizard_stream_t*) Lizard_streamPtr;
if (dictSize > LIZARD_DICT_SIZE) {
dictionary += dictSize - LIZARD_DICT_SIZE;
dictSize = LIZARD_DICT_SIZE;
}
Lizard_init(ctxPtr, (const BYTE*)dictionary);
if (dictSize >= HASH_UPDATE_LIMIT) Lizard_Insert (ctxPtr, (const BYTE*)dictionary + (dictSize - (HASH_UPDATE_LIMIT-1)));
ctxPtr->end = (const BYTE*)dictionary + dictSize;
return dictSize;
}
static void Lizard_setExternalDict(Lizard_stream_t* ctxPtr, const BYTE* newBlock)
{
if (ctxPtr->end >= ctxPtr->base + HASH_UPDATE_LIMIT) Lizard_Insert (ctxPtr, ctxPtr->end - (HASH_UPDATE_LIMIT-1)); /* Referencing remaining dictionary content */
/* Only one memory segment for extDict, so any previous extDict is lost at this stage */
ctxPtr->lowLimit = ctxPtr->dictLimit;
ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
ctxPtr->dictBase = ctxPtr->base;
ctxPtr->base = newBlock - ctxPtr->dictLimit;
ctxPtr->end = newBlock;
ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
}
/* dictionary saving */
int Lizard_saveDict (Lizard_stream_t* Lizard_streamPtr, char* safeBuffer, int dictSize)
{
Lizard_stream_t* const ctx = (Lizard_stream_t*)Lizard_streamPtr;
int const prefixSize = (int)(ctx->end - (ctx->base + ctx->dictLimit));
if (dictSize > LIZARD_DICT_SIZE) dictSize = LIZARD_DICT_SIZE;
if (dictSize < 4) dictSize = 0;
if (dictSize > prefixSize) dictSize = prefixSize;
memmove(safeBuffer, ctx->end - dictSize, dictSize);
{ U32 const endIndex = (U32)(ctx->end - ctx->base);
ctx->end = (const BYTE*)safeBuffer + dictSize;
ctx->base = ctx->end - endIndex;
ctx->dictLimit = endIndex - dictSize;
ctx->lowLimit = endIndex - dictSize;
if (ctx->nextToUpdate < ctx->dictLimit) ctx->nextToUpdate = ctx->dictLimit;
}
return dictSize;
}
FORCE_INLINE int Lizard_compress_generic (
void* ctxvoid,
const char* source,
char* dest,
int inputSize,
int maxOutputSize)
{
Lizard_stream_t* ctx = (Lizard_stream_t*) ctxvoid;
size_t dictSize = (size_t)(ctx->end - ctx->base) - ctx->dictLimit;
const BYTE* ip = (const BYTE*) source;
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + maxOutputSize;
int res;
(void)dictSize;
LIZARD_LOG_COMPRESS("Lizard_compress_generic source=%p inputSize=%d dest=%p maxOutputSize=%d cLevel=%d dictBase=%p dictSize=%d\n", source, inputSize, dest, maxOutputSize, ctx->compressionLevel, ctx->dictBase, (int)dictSize);
*op++ = (BYTE)ctx->compressionLevel;
maxOutputSize--; // can be lower than 0
ctx->end += inputSize;
ctx->srcBase = ctx->off24pos = ip;
ctx->destBase = (BYTE*)dest;
while (inputSize > 0)
{
int inputPart = MIN(LIZARD_BLOCK_SIZE, inputSize);
if (ctx->huffType) Lizard_rescaleFreqs(ctx);
Lizard_initBlock(ctx);
ctx->diffBase = ip;
switch(ctx->params.parserType)
{
default:
case Lizard_parser_fastSmall:
res = Lizard_compress_fastSmall(ctx, ip, ip+inputPart); break;
case Lizard_parser_fast:
res = Lizard_compress_fast(ctx, ip, ip+inputPart); break;
case Lizard_parser_noChain:
res = Lizard_compress_noChain(ctx, ip, ip+inputPart); break;
case Lizard_parser_hashChain:
res = Lizard_compress_hashChain(ctx, ip, ip+inputPart); break;
#ifndef USE_LZ4_ONLY
case Lizard_parser_fastBig:
res = Lizard_compress_fastBig(ctx, ip, ip+inputPart); break;
case Lizard_parser_priceFast:
res = Lizard_compress_priceFast(ctx, ip, ip+inputPart); break;
case Lizard_parser_lowestPrice:
res = Lizard_compress_lowestPrice(ctx, ip, ip+inputPart); break;
case Lizard_parser_optimalPrice:
case Lizard_parser_optimalPriceBT:
res = Lizard_compress_optimalPrice(ctx, ip, ip+inputPart); break;
#else
case Lizard_parser_priceFast:
case Lizard_parser_lowestPrice:
case Lizard_parser_optimalPrice:
case Lizard_parser_optimalPriceBT:
res = 0;
#endif
}
LIZARD_LOG_COMPRESS("Lizard_compress_generic res=%d inputPart=%d \n", res, inputPart);
if (res <= 0) return res;
if (Lizard_writeBlock(ctx, ip, inputPart, &op, oend)) goto _output_error;
ip += inputPart;
inputSize -= inputPart;
LIZARD_LOG_COMPRESS("Lizard_compress_generic in=%d out=%d\n", (int)(ip-(const BYTE*)source), (int)(op-(BYTE*)dest));
}
LIZARD_LOG_COMPRESS("Lizard_compress_generic total=%d\n", (int)(op-(BYTE*)dest));
return (int)(op-(BYTE*)dest);
_output_error:
LIZARD_LOG_COMPRESS("Lizard_compress_generic ERROR\n");
return 0;
}
int Lizard_compress_continue (Lizard_stream_t* ctxPtr,
const char* source, char* dest,
int inputSize, int maxOutputSize)
{
/* auto-init if required */
if (ctxPtr->base == NULL) Lizard_init(ctxPtr, (const BYTE*) source);
/* Check overflow */
if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) {
size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
if (dictSize > LIZARD_DICT_SIZE) dictSize = LIZARD_DICT_SIZE;
Lizard_loadDict((Lizard_stream_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize);
}
/* Check if blocks follow each other */
if ((const BYTE*)source != ctxPtr->end)
Lizard_setExternalDict(ctxPtr, (const BYTE*)source);
/* Check overlapping input/dictionary space */
{ const BYTE* sourceEnd = (const BYTE*) source + inputSize;
const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) {
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
}
}
return Lizard_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize);
}
int Lizard_compress_extState (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
{
Lizard_stream_t* ctx = (Lizard_stream_t*) state;
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
/* initialize stream */
Lizard_initStream(ctx, compressionLevel);
Lizard_init((Lizard_stream_t*)state, (const BYTE*)src);
return Lizard_compress_generic (state, src, dst, srcSize, maxDstSize);
}
int Lizard_compress(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
{
int cSize;
Lizard_stream_t* statePtr = Lizard_createStream(compressionLevel);
if (!statePtr) return 0;
cSize = Lizard_compress_extState(statePtr, src, dst, srcSize, maxDstSize, compressionLevel);
Lizard_freeStream(statePtr);
return cSize;
}
/**************************************
* Level1 functions
**************************************/
int Lizard_compress_extState_MinLevel(void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
{
return Lizard_compress_extState(state, source, dest, inputSize, maxOutputSize, LIZARD_MIN_CLEVEL);
}
int Lizard_compress_MinLevel(const char* source, char* dest, int inputSize, int maxOutputSize)
{
return Lizard_compress(source, dest, inputSize, maxOutputSize, LIZARD_MIN_CLEVEL);
}
Lizard_stream_t* Lizard_createStream_MinLevel(void)
{
return Lizard_createStream(LIZARD_MIN_CLEVEL);
}
Lizard_stream_t* Lizard_resetStream_MinLevel(Lizard_stream_t* Lizard_stream)
{
return Lizard_resetStream (Lizard_stream, LIZARD_MIN_CLEVEL);
}

208
contrib/lizard/lib/lizard_compress.h Normal file
View File

@ -0,0 +1,208 @@
/*
Lizard - Fast LZ compression algorithm
Header File
Copyright (C) 2011-2016, Yann Collet
Copyright (C) 2016-2017, Przemyslaw Skibinski <inikep@gmail.com>
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Lizard source repository : https://github.com/inikep/lizard
*/
#ifndef LIZARD_H_2983
#define LIZARD_H_2983
#if defined (__cplusplus)
extern "C" {
#endif
/*
* lizard_compress.h provides block compression functions. It gives full buffer control to user.
* Block compression functions are not-enough to send information,
* since it's still necessary to provide metadata (such as compressed size),
* and each application can do it in whichever way it wants.
* For interoperability, there is Lizard frame specification (lizard_Frame_format.md).
* A library is provided to take care of it, see lizard_frame.h.
*/
/*^***************************************************************
* Export parameters
*****************************************************************/
/*
* LIZARD_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
*/
#if defined(LIZARD_DLL_EXPORT) && (LIZARD_DLL_EXPORT==1)
# define LIZARDLIB_API __declspec(dllexport)
#elif defined(LIZARD_DLL_IMPORT) && (LIZARD_DLL_IMPORT==1)
# define LIZARDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define LIZARDLIB_API
#endif
/*-************************************
* Version
**************************************/
#define LIZARD_VERSION_MAJOR 1 /* for breaking interface changes */
#define LIZARD_VERSION_MINOR 0 /* for new (non-breaking) interface capabilities */
#define LIZARD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
#define LIZARD_VERSION_NUMBER (LIZARD_VERSION_MAJOR *100*100 + LIZARD_VERSION_MINOR *100 + LIZARD_VERSION_RELEASE)
int Lizard_versionNumber (void);
#define LIZARD_LIB_VERSION LIZARD_VERSION_MAJOR.LIZARD_VERSION_MINOR.LIZARD_VERSION_RELEASE
#define LIZARD_QUOTE(str) #str
#define LIZARD_EXPAND_AND_QUOTE(str) LIZARD_QUOTE(str)
#define LIZARD_VERSION_STRING LIZARD_EXPAND_AND_QUOTE(LIZARD_LIB_VERSION)
const char* Lizard_versionString (void);
typedef struct Lizard_stream_s Lizard_stream_t;
#define LIZARD_MIN_CLEVEL 10 /* minimum compression level */
#ifndef LIZARD_NO_HUFFMAN
#define LIZARD_MAX_CLEVEL 49 /* maximum compression level */
#else
#define LIZARD_MAX_CLEVEL 29 /* maximum compression level */
#endif
#define LIZARD_DEFAULT_CLEVEL 17
/*-************************************
* Simple Functions
**************************************/
LIZARDLIB_API int Lizard_compress (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
/*
Lizard_compress() :
Compresses 'sourceSize' bytes from buffer 'source'
into already allocated 'dest' buffer of size 'maxDestSize'.
Compression is guaranteed to succeed if 'maxDestSize' >= Lizard_compressBound(sourceSize).
It also runs faster, so it's a recommended setting.
If the function cannot compress 'source' into a more limited 'dest' budget,
compression stops *immediately*, and the function result is zero.
As a consequence, 'dest' content is not valid.
This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
sourceSize : Max supported value is LIZARD_MAX_INPUT_VALUE
maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
or 0 if compression fails
*/
/*-************************************
* Advanced Functions
**************************************/
#define LIZARD_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
#define LIZARD_BLOCK_SIZE (1<<17)
#define LIZARD_BLOCK64K_SIZE (1<<16)
#define LIZARD_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LIZARD_MAX_INPUT_SIZE ? 0 : (isize) + 1 + 1 + ((isize/LIZARD_BLOCK_SIZE)+1)*4)
/*!
Lizard_compressBound() :
Provides the maximum size that Lizard compression may output in a "worst case" scenario (input data not compressible)
This function is primarily useful for memory allocation purposes (destination buffer size).
Macro LIZARD_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
Note that Lizard_compress() compress faster when dest buffer size is >= Lizard_compressBound(srcSize)
inputSize : max supported value is LIZARD_MAX_INPUT_SIZE
return : maximum output size in a "worst case" scenario
or 0, if input size is too large ( > LIZARD_MAX_INPUT_SIZE)
*/
LIZARDLIB_API int Lizard_compressBound(int inputSize);
/*!
Lizard_compress_extState() :
Same compression function, just using an externally allocated memory space to store compression state.
Use Lizard_sizeofState() to know how much memory must be allocated,
and allocate it on 8-bytes boundaries (using malloc() typically).
Then, provide it as 'void* state' to compression function.
*/
LIZARDLIB_API int Lizard_sizeofState(int compressionLevel);
LIZARDLIB_API int Lizard_compress_extState(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
/*-*********************************************
* Streaming Compression Functions
***********************************************/
/*! Lizard_createStream() will allocate and initialize an `Lizard_stream_t` structure.
* Lizard_freeStream() releases its memory.
* In the context of a DLL (liblizard), please use these methods rather than the static struct.
* They are more future proof, in case of a change of `Lizard_stream_t` size.
*/
LIZARDLIB_API Lizard_stream_t* Lizard_createStream(int compressionLevel);
LIZARDLIB_API int Lizard_freeStream (Lizard_stream_t* streamPtr);
/*! Lizard_resetStream() :
* Use this function to reset/reuse an allocated `Lizard_stream_t` structure
*/
LIZARDLIB_API Lizard_stream_t* Lizard_resetStream (Lizard_stream_t* streamPtr, int compressionLevel);
/*! Lizard_loadDict() :
* Use this function to load a static dictionary into Lizard_stream.
* Any previous data will be forgotten, only 'dictionary' will remain in memory.
* Loading a size of 0 is allowed.
* Return : dictionary size, in bytes (necessarily <= LIZARD_DICT_SIZE)
*/
LIZARDLIB_API int Lizard_loadDict (Lizard_stream_t* streamPtr, const char* dictionary, int dictSize);
/*! Lizard_compress_continue() :
* Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
* Important : Previous data blocks are assumed to still be present and unmodified !
* 'dst' buffer must be already allocated.
* If maxDstSize >= Lizard_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
* If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
*/
LIZARDLIB_API int Lizard_compress_continue (Lizard_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize);
/*! Lizard_saveDict() :
* If previously compressed data block is not guaranteed to remain available at its memory location,
* save it into a safer place (char* safeBuffer).
* Note : you don't need to call Lizard_loadDict() afterwards,
* dictionary is immediately usable, you can therefore call Lizard_compress_continue().
* Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
*/
LIZARDLIB_API int Lizard_saveDict (Lizard_stream_t* streamPtr, char* safeBuffer, int dictSize);
#if defined (__cplusplus)
}
#endif
#endif /* LIZARD_H_2983827168210 */

301
contrib/lizard/lib/lizard_compress_liz.h Normal file
View File

@ -0,0 +1,301 @@
#define LIZARD_FREQ_DIV 5
FORCE_INLINE void Lizard_setLog2Prices(Lizard_stream_t* ctx)
{
ctx->log2LitSum = Lizard_highbit32(ctx->litSum+1);
ctx->log2FlagSum = Lizard_highbit32(ctx->flagSum+1);
}
MEM_STATIC void Lizard_rescaleFreqs(Lizard_stream_t* ctx)
{
unsigned u;
ctx->cachedLiterals = NULL;
ctx->cachedPrice = ctx->cachedLitLength = 0;
ctx->litPriceSum = 0;
if (ctx->litSum == 0) {
ctx->litSum = 2 * 256;
ctx->flagSum = 2 * 256;
for (u=0; u < 256; u++) {
ctx->litFreq[u] = 2;
ctx->flagFreq[u] = 2;
}
} else {
ctx->litSum = 0;
ctx->flagSum = 0;
for (u=0; u < 256; u++) {
ctx->litFreq[u] = 1 + (ctx->litFreq[u]>>LIZARD_FREQ_DIV);
ctx->litSum += ctx->litFreq[u];
ctx->flagFreq[u] = 1 + (ctx->flagFreq[u]>>LIZARD_FREQ_DIV);
ctx->flagSum += ctx->flagFreq[u];
}
}
Lizard_setLog2Prices(ctx);
}
FORCE_INLINE int Lizard_encodeSequence_LIZv1 (
Lizard_stream_t* ctx,
const BYTE** ip,
const BYTE** anchor,
size_t matchLength,
const BYTE* const match)
{
U32 offset = (U32)(*ip - match);
size_t length = (size_t)(*ip - *anchor);
BYTE* token = (ctx->flagsPtr)++;
if (length > 0 || offset < LIZARD_MAX_16BIT_OFFSET) {
/* Encode Literal length */
// if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - length - LIZARD_LENGTH_SIZE_LIZv1(length) - WILDCOPYLENGTH)) { LIZARD_LOG_COMPRESS_LIZv1("encodeSequence overflow1\n"); return 1; } /* Check output limit */
if (length >= MAX_SHORT_LITLEN)
{ size_t len;
*token = MAX_SHORT_LITLEN;
len = length - MAX_SHORT_LITLEN;
if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; }
else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; }
else *(ctx->literalsPtr)++ = (BYTE)len;
}
else *token = (BYTE)length;
/* Copy Literals */
Lizard_wildCopy(ctx->literalsPtr, *anchor, (ctx->literalsPtr) + length);
#ifndef LIZARD_NO_HUFFMAN
if (ctx->huffType) {
ctx->litSum += (U32)length;
ctx->litPriceSum += (U32)(length * ctx->log2LitSum);
{ U32 u;
for (u=0; u < length; u++) {
ctx->litPriceSum -= Lizard_highbit32(ctx->litFreq[ctx->literalsPtr[u]]+1);
ctx->litFreq[ctx->literalsPtr[u]]++;
} }
}
#endif
ctx->literalsPtr += length;
if (offset >= LIZARD_MAX_16BIT_OFFSET) {
COMPLOG_CODEWORDS_LIZv1("T32+ literal=%u match=%u offset=%d\n", (U32)length, 0, 0);
*token+=(1<<ML_RUN_BITS);
#ifndef LIZARD_NO_HUFFMAN
if (ctx->huffType) {
ctx->flagFreq[*token]++;
ctx->flagSum++;
}
#endif
token = (ctx->flagsPtr)++;
}
}
/* Encode Offset */
if (offset >= LIZARD_MAX_16BIT_OFFSET) // 24-bit offset
{
if (matchLength < MM_LONGOFF) printf("ERROR matchLength=%d/%d\n", (int)matchLength, MM_LONGOFF), exit(1);
// if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - 8 /*LIZARD_LENGTH_SIZE_LIZv1(length)*/)) { LIZARD_LOG_COMPRESS_LIZv1("encodeSequence overflow2\n"); return 1; } /* Check output limit */
if (matchLength - MM_LONGOFF >= LIZARD_LAST_LONG_OFF)
{
size_t len = matchLength - MM_LONGOFF - LIZARD_LAST_LONG_OFF;
*token = LIZARD_LAST_LONG_OFF;
if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; }
else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; }
else *(ctx->literalsPtr)++ = (BYTE)len;
COMPLOG_CODEWORDS_LIZv1("T31 literal=%u match=%u offset=%d\n", 0, (U32)matchLength, offset);
}
else
{
COMPLOG_CODEWORDS_LIZv1("T0-30 literal=%u match=%u offset=%d\n", 0, (U32)matchLength, offset);
*token = (BYTE)(matchLength - MM_LONGOFF);
}
MEM_writeLE24(ctx->offset24Ptr, offset);
ctx->offset24Ptr += 3;
ctx->last_off = offset;
ctx->off24pos = *ip;
}
else
{
COMPLOG_CODEWORDS_LIZv1("T32+ literal=%u match=%u offset=%d\n", (U32)length, (U32)matchLength, offset);
if (offset == 0)
{
*token+=(1<<ML_RUN_BITS);
}
else
{
// it should never happen
if (offset < 8) { printf("ERROR offset=%d\n", (int)offset); exit(1); }
if (matchLength < MINMATCH) { printf("ERROR matchLength[%d] < MINMATCH offset=%d\n", (int)matchLength, (int)ctx->last_off); exit(1); }
ctx->last_off = offset;
MEM_writeLE16(ctx->offset16Ptr, (U16)ctx->last_off); ctx->offset16Ptr += 2;
}
/* Encode MatchLength */
length = matchLength;
// if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - 5 /*LIZARD_LENGTH_SIZE_LIZv1(length)*/)) { LIZARD_LOG_COMPRESS_LIZv1("encodeSequence overflow2\n"); return 1; } /* Check output limit */
if (length >= MAX_SHORT_MATCHLEN) {
*token += (BYTE)(MAX_SHORT_MATCHLEN<<RUN_BITS_LIZv1);
length -= MAX_SHORT_MATCHLEN;
if (length >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(length)); ctx->literalsPtr += 4; }
else if (length >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(length)); ctx->literalsPtr += 3; }
else *(ctx->literalsPtr)++ = (BYTE)length;
}
else *token += (BYTE)(length<<RUN_BITS_LIZv1);
}
#ifndef LIZARD_NO_HUFFMAN
if (ctx->huffType) {
ctx->flagFreq[*token]++;
ctx->flagSum++;
Lizard_setLog2Prices(ctx);
}
#endif
/* Prepare next loop */
*ip += matchLength;
*anchor = *ip;
return 0;
}
FORCE_INLINE int Lizard_encodeLastLiterals_LIZv1 (
Lizard_stream_t* ctx,
const BYTE** ip,
const BYTE** anchor)
{
size_t length = (int)(*ip - *anchor);
(void)ctx;
memcpy(ctx->literalsPtr, *anchor, length);
ctx->literalsPtr += length;
return 0;
}
#define LIZARD_PRICE_MULT 1
#define LIZARD_GET_TOKEN_PRICE_LIZv1(token) (LIZARD_PRICE_MULT * (ctx->log2FlagSum - Lizard_highbit32(ctx->flagFreq[token]+1)))
FORCE_INLINE size_t Lizard_get_price_LIZv1(Lizard_stream_t* const ctx, int rep, const BYTE *ip, const BYTE *off24pos, size_t litLength, U32 offset, size_t matchLength)
{
size_t price = 0;
BYTE token = 0;
#ifndef LIZARD_NO_HUFFMAN
const BYTE* literals = ip - litLength;
U32 u;
if ((ctx->huffType) && (ctx->params.parserType != Lizard_parser_lowestPrice)) {
if (ctx->cachedLiterals == literals && litLength >= ctx->cachedLitLength) {
size_t const additional = litLength - ctx->cachedLitLength;
const BYTE* literals2 = ctx->cachedLiterals + ctx->cachedLitLength;
price = ctx->cachedPrice + LIZARD_PRICE_MULT * additional * ctx->log2LitSum;
for (u=0; u < additional; u++)
price -= LIZARD_PRICE_MULT * Lizard_highbit32(ctx->litFreq[literals2[u]]+1);
ctx->cachedPrice = (U32)price;
ctx->cachedLitLength = (U32)litLength;
} else {
price = LIZARD_PRICE_MULT * litLength * ctx->log2LitSum;
for (u=0; u < litLength; u++)
price -= LIZARD_PRICE_MULT * Lizard_highbit32(ctx->litFreq[literals[u]]+1);
if (litLength >= 12) {
ctx->cachedLiterals = literals;
ctx->cachedPrice = (U32)price;
ctx->cachedLitLength = (U32)litLength;
}
}
}
else
price += 8*litLength; /* Copy Literals */
#else
price += 8*litLength; /* Copy Literals */
(void)ip;
(void)ctx;
#endif
(void)off24pos;
(void)rep;
if (litLength > 0 || offset < LIZARD_MAX_16BIT_OFFSET) {
/* Encode Literal length */
if (litLength >= MAX_SHORT_LITLEN)
{ size_t len = litLength - MAX_SHORT_LITLEN;
token = MAX_SHORT_LITLEN;
if (len >= (1<<16)) price += 32;
else if (len >= 254) price += 24;
else price += 8;
}
else token = (BYTE)litLength;
if (offset >= LIZARD_MAX_16BIT_OFFSET) {
token+=(1<<ML_RUN_BITS);
if (ctx->huffType && ctx->params.parserType != Lizard_parser_lowestPrice)
price += LIZARD_GET_TOKEN_PRICE_LIZv1(token);
else
price += 8;
}
}
/* Encode Offset */
if (offset >= LIZARD_MAX_16BIT_OFFSET) { // 24-bit offset
if (matchLength < MM_LONGOFF) return LIZARD_MAX_PRICE; // error
if (matchLength - MM_LONGOFF >= LIZARD_LAST_LONG_OFF) {
size_t len = matchLength - MM_LONGOFF - LIZARD_LAST_LONG_OFF;
token = LIZARD_LAST_LONG_OFF;
if (len >= (1<<16)) price += 32;
else if (len >= 254) price += 24;
else price += 8;
} else {
token = (BYTE)(matchLength - MM_LONGOFF);
}
price += 24;
} else {
size_t length;
if (offset == 0) {
token+=(1<<ML_RUN_BITS);
} else {
if (offset < 8) return LIZARD_MAX_PRICE; // error
if (matchLength < MINMATCH) return LIZARD_MAX_PRICE; // error
price += 16;
}
/* Encode MatchLength */
length = matchLength;
if (length >= MAX_SHORT_MATCHLEN) {
token += (BYTE)(MAX_SHORT_MATCHLEN<<RUN_BITS_LIZv1);
length -= MAX_SHORT_MATCHLEN;
if (length >= (1<<16)) price += 32;
else if (length >= 254) price += 24;
else price += 8;
}
else token += (BYTE)(length<<RUN_BITS_LIZv1);
}
if (offset > 0 || matchLength > 0) {
int offset_load = Lizard_highbit32(offset);
if (ctx->huffType) {
price += ((offset_load>=20) ? ((offset_load-19)*4) : 0);
price += 4 + (matchLength==1);
} else {
price += ((offset_load>=16) ? ((offset_load-15)*4) : 0);
price += 6 + (matchLength==1);
}
if (ctx->huffType && ctx->params.parserType != Lizard_parser_lowestPrice)
price += LIZARD_GET_TOKEN_PRICE_LIZv1(token);
else
price += 8;
} else {
if (ctx->huffType && ctx->params.parserType != Lizard_parser_lowestPrice)
price += LIZARD_GET_TOKEN_PRICE_LIZv1(token); // 1=better ratio
}
return price;
}

162
contrib/lizard/lib/lizard_compress_lz4.h Normal file
View File

@ -0,0 +1,162 @@
#define LIZARD_LENGTH_SIZE_LZ4(len) ((len >= (1<<16)+RUN_MASK_LZ4) ? 5 : ((len >= 254+RUN_MASK_LZ4) ? 3 : ((len >= RUN_MASK_LZ4) ? 1 : 0)))
FORCE_INLINE int Lizard_encodeSequence_LZ4 (
Lizard_stream_t* ctx,
const BYTE** ip,
const BYTE** anchor,
size_t matchLength,
const BYTE* const match)
{
size_t length = (size_t)(*ip - *anchor);
BYTE* token = (ctx->flagsPtr)++;
(void) ctx;
COMPLOG_CODEWORDS_LZ4("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
/* Encode Literal length */
// if (ctx->literalsPtr > ctx->literalsEnd - length - LIZARD_LENGTH_SIZE_LZ4(length) - 2 - WILDCOPYLENGTH) { LIZARD_LOG_COMPRESS_LZ4("encodeSequence overflow1\n"); return 1; } /* Check output limit */
if (length >= RUN_MASK_LZ4)
{ size_t len = length - RUN_MASK_LZ4;
*token = RUN_MASK_LZ4;
if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; }
else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; }
else *(ctx->literalsPtr)++ = (BYTE)len;
}
else *token = (BYTE)length;
/* Copy Literals */
if (length > 0) {
Lizard_wildCopy(ctx->literalsPtr, *anchor, (ctx->literalsPtr) + length);
#if 0 //def LIZARD_USE_HUFFMAN
ctx->litSum += (U32)length;
ctx->litPriceSum += (U32)(length * ctx->log2LitSum);
{ U32 u;
for (u=0; u < length; u++) {
ctx->litPriceSum -= Lizard_highbit32(ctx->litFreq[ctx->literalsPtr[u]]+1);
ctx->litFreq[ctx->literalsPtr[u]]++;
} }
#endif
ctx->literalsPtr += length;
}
/* Encode Offset */
MEM_writeLE16(ctx->literalsPtr, (U16)(*ip-match));
ctx->literalsPtr+=2;
/* Encode MatchLength */
length = matchLength - MINMATCH;
// if (ctx->literalsPtr > ctx->literalsEnd - 5 /*LIZARD_LENGTH_SIZE_LZ4(length)*/) { LIZARD_LOG_COMPRESS_LZ4("encodeSequence overflow2\n"); return 1; } /* Check output limit */
if (length >= ML_MASK_LZ4) {
*token += (BYTE)(ML_MASK_LZ4<<RUN_BITS_LZ4);
length -= ML_MASK_LZ4;
if (length >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(length)); ctx->literalsPtr += 4; }
else if (length >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(length)); ctx->literalsPtr += 3; }
else *(ctx->literalsPtr)++ = (BYTE)length;
}
else *token += (BYTE)(length<<RUN_BITS_LZ4);
#ifndef LIZARD_NO_HUFFMAN
if (ctx->huffType) {
ctx->flagFreq[*token]++;
ctx->flagSum++;
Lizard_setLog2Prices(ctx);
}
#endif
/* Prepare next loop */
*ip += matchLength;
*anchor = *ip;
return 0;
}
FORCE_INLINE int Lizard_encodeLastLiterals_LZ4 (
Lizard_stream_t* ctx,
const BYTE** ip,
const BYTE** anchor)
{
size_t length = (int)(*ip - *anchor);
(void)ctx;
memcpy(ctx->literalsPtr, *anchor, length);
ctx->literalsPtr += length;
return 0;
}
#define LIZARD_GET_TOKEN_PRICE_LZ4(token) (ctx->log2FlagSum - Lizard_highbit32(ctx->flagFreq[token]+1))
FORCE_INLINE size_t Lizard_get_price_LZ4(Lizard_stream_t* const ctx, const BYTE *ip, const size_t litLength, U32 offset, size_t matchLength)
{
size_t price = 0;
BYTE token = 0;
#if 0 //def LIZARD_USE_HUFFMAN
const BYTE* literals = ip - litLength;
U32 u;
if (ctx->cachedLiterals == literals && litLength >= ctx->cachedLitLength) {
size_t const additional = litLength - ctx->cachedLitLength;
const BYTE* literals2 = ctx->cachedLiterals + ctx->cachedLitLength;
price = ctx->cachedPrice + additional * ctx->log2LitSum;
for (u=0; u < additional; u++)
price -= Lizard_highbit32(ctx->litFreq[literals2[u]]+1);
ctx->cachedPrice = (U32)price;
ctx->cachedLitLength = (U32)litLength;
} else {
price = litLength * ctx->log2LitSum;
for (u=0; u < litLength; u++)
price -= Lizard_highbit32(ctx->litFreq[literals[u]]+1);
if (litLength >= 12) {
ctx->cachedLiterals = literals;
ctx->cachedPrice = (U32)price;
ctx->cachedLitLength = (U32)litLength;
}
}
#else
price += 8*litLength; /* Copy Literals */
(void)ip;
(void)ctx;
#endif
/* Encode Literal length */
if (litLength >= RUN_MASK_LZ4) {
size_t len = litLength - RUN_MASK_LZ4;
token = RUN_MASK_LZ4;
if (len >= (1<<16)) price += 32;
else if (len >= 254) price += 24;
else price += 8;
}
else token = (BYTE)litLength;
/* Encode MatchLength */
if (offset) {
size_t length;
price += 16; /* Encode Offset */
if (offset < 8) return LIZARD_MAX_PRICE; // error
if (matchLength < MINMATCH) return LIZARD_MAX_PRICE; // error
length = matchLength - MINMATCH;
if (length >= ML_MASK_LZ4) {
token += (BYTE)(ML_MASK_LZ4<<RUN_BITS_LZ4);
length -= ML_MASK_LZ4;
if (length >= (1<<16)) price += 32;
else if (length >= 254) price += 24;
else price += 8;
}
else token += (BYTE)(length<<RUN_BITS_LZ4);
}
if (ctx->huffType) {
if (offset > 0 || matchLength > 0) price += 2;
price += LIZARD_GET_TOKEN_PRICE_LZ4(token);
} else {
price += 8; // token
}
return price;
}

372
contrib/lizard/lib/lizard_decompress.c Normal file
View File

@ -0,0 +1,372 @@
/*
Lizard - Fast LZ compression algorithm
Copyright (C) 2011-2016, Yann Collet
Copyright (C) 2016-2017, Przemyslaw Skibinski <inikep@gmail.com>
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Lizard source repository : https://github.com/inikep/lizard
*/
/**************************************
* Includes
**************************************/
//#define LIZARD_STATS 1 // 0=simple stats, 1=more, 2=full
#ifdef LIZARD_STATS
#include "test/lizard_stats.h"
#endif
#include "lizard_compress.h"
#include "lizard_decompress.h"
#include "lizard_common.h"
#include <stdio.h> // printf
#include <stdint.h> // intptr_t
/*-************************************
* Local Structures and types
**************************************/
typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive;
typedef enum { full = 0, partial = 1 } earlyEnd_directive;
#include "lizard_decompress_lz4.h"
#ifndef USE_LZ4_ONLY
#ifdef LIZARD_USE_TEST
#include "test/lizard_common_test.h"
#include "test/lizard_decompress_test.h"
#else
#include "lizard_decompress_liz.h"
#endif
#endif
#include "entropy/huf.h"
/*-*****************************
* Decompression functions
*******************************/
FORCE_INLINE size_t Lizard_readStream(int flag, const BYTE** ip, const BYTE* const iend, BYTE* op, BYTE* const oend, const BYTE** streamPtr, const BYTE** streamEnd, int streamFlag)
{
if (!flag) {
if (*ip > iend - 3) return 0;
*streamPtr = *ip + 3;
*streamEnd = *streamPtr + MEM_readLE24(*ip);
if (*streamEnd < *streamPtr) return 0;
*ip = *streamEnd;
#ifdef LIZARD_STATS
uncompr_stream[streamFlag] += *streamEnd-*streamPtr;
#else
(void)streamFlag;
#endif
return 1;
} else {
#ifndef LIZARD_NO_HUFFMAN
size_t res, streamLen, comprStreamLen;
if (*ip > iend - 6) return 0;
streamLen = MEM_readLE24(*ip);
comprStreamLen = MEM_readLE24(*ip + 3);
if ((op > oend - streamLen) || (*ip + comprStreamLen > iend - 6)) return 0;
res = HUF_decompress(op, streamLen, *ip + 6, comprStreamLen);
if (HUF_isError(res) || (res != streamLen)) return 0;
*ip += comprStreamLen + 6;
*streamPtr = op;
*streamEnd = *streamPtr + streamLen;
#ifdef LIZARD_STATS
compr_stream[streamFlag] += comprStreamLen + 6;
decompr_stream[streamFlag] += *streamEnd-*streamPtr;
#endif
return 1;
#else
fprintf(stderr, "compiled with LIZARD_NO_HUFFMAN\n");
(void)op; (void)oend;
return 0;
#endif
}
}
FORCE_INLINE int Lizard_decompress_generic(
const char* source,
char* const dest,
int inputSize,
int outputSize, /* this value is the max size of Output Buffer. */
int partialDecoding, /* full, partial */
int targetOutputSize, /* only used if partialDecoding==partial */
int dict, /* noDict, withPrefix64k, usingExtDict */
const BYTE* const lowPrefix, /* == dest if dict == noDict */
const BYTE* const dictStart, /* only if dict==usingExtDict */
const size_t dictSize /* note : = 0 if noDict */
)
{
/* Local Variables */
const BYTE* ip = (const BYTE*) source, *istart = (const BYTE*) source;
const BYTE* const iend = ip + inputSize;
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + outputSize;
BYTE* oexit = op + targetOutputSize;
Lizard_parameters params;
Lizard_dstream_t ctx;
BYTE* decompFlagsBase, *decompOff24Base, *decompOff16Base, *decompLiteralsBase = NULL;
int res, compressionLevel;
if (inputSize < 1) { LIZARD_LOG_DECOMPRESS("inputSize=%d outputSize=%d targetOutputSize=%d partialDecoding=%d\n", inputSize, outputSize, targetOutputSize, partialDecoding); return 0; }
compressionLevel = *ip++;
if (compressionLevel < LIZARD_MIN_CLEVEL || compressionLevel > LIZARD_MAX_CLEVEL) {
LIZARD_LOG_DECOMPRESS("ERROR Lizard_decompress_generic inputSize=%d compressionLevel=%d\n", inputSize, compressionLevel);
return -1;
}
LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic ip=%p inputSize=%d targetOutputSize=%d dest=%p outputSize=%d cLevel=%d dict=%d dictSize=%d dictStart=%p partialDecoding=%d\n", ip, inputSize, targetOutputSize, dest, outputSize, compressionLevel, dict, (int)dictSize, dictStart, partialDecoding);
decompLiteralsBase = (BYTE*)malloc(4*LIZARD_HUF_BLOCK_SIZE);
if (!decompLiteralsBase) return -1;
decompFlagsBase = decompLiteralsBase + LIZARD_HUF_BLOCK_SIZE;
decompOff24Base = decompFlagsBase + LIZARD_HUF_BLOCK_SIZE;
decompOff16Base = decompOff24Base + LIZARD_HUF_BLOCK_SIZE;
#ifdef LIZARD_STATS
init_stats();
#endif
(void)istart;
while (ip < iend)
{
res = *ip++;
if (res == LIZARD_FLAG_UNCOMPRESSED) /* uncompressed */
{
uint32_t length;
if (ip > iend - 3) { LIZARD_LOG_DECOMPRESS("UNCOMPRESSED ip[%p] > iend[%p] - 3\n", ip, iend); goto _output_error; }
length = MEM_readLE24(ip);
ip += 3;
// printf("%d: total=%d block=%d UNCOMPRESSED op=%p oexit=%p oend=%p\n", (int)(op-(BYTE*)dest) ,(int)(ip-istart), length, op, oexit, oend);
if (ip + length > iend || op + length > oend) { LIZARD_LOG_DECOMPRESS("UNCOMPRESSED ip[%p]+length[%d] > iend[%p]\n", ip, length, iend); goto _output_error; }
memcpy(op, ip, length);
op += length;
ip += length;
if ((partialDecoding) && (op >= oexit)) break;
#ifdef LIZARD_STATS
uncompr_stream[LIZARD_STREAM_UNCOMPRESSED] += length;
#endif
continue;
}
if (res&LIZARD_FLAG_LEN) {
LIZARD_LOG_DECOMPRESS("res=%d\n", res); goto _output_error;
}
if (ip > iend - 5*3) goto _output_error;
ctx.lenPtr = (const BYTE*)ip + 3;
ctx.lenEnd = ctx.lenPtr + MEM_readLE24(ip);
if (ctx.lenEnd < ctx.lenPtr || (ctx.lenEnd > iend - 3)) goto _output_error;
#ifdef LIZARD_STATS
uncompr_stream[LIZARD_STREAM_LEN] += ctx.lenEnd-ctx.lenPtr + 3;
#endif
ip = ctx.lenEnd;
{ size_t streamLen;
#ifdef LIZARD_USE_LOGS
const BYTE* ipos;
size_t comprFlagsLen, comprLiteralsLen, total;
#endif
streamLen = Lizard_readStream(res&LIZARD_FLAG_OFFSET16, &ip, iend, decompOff16Base, decompOff16Base + LIZARD_HUF_BLOCK_SIZE, &ctx.offset16Ptr, &ctx.offset16End, LIZARD_STREAM_OFFSET16);
if (streamLen == 0) goto _output_error;
streamLen = Lizard_readStream(res&LIZARD_FLAG_OFFSET24, &ip, iend, decompOff24Base, decompOff24Base + LIZARD_HUF_BLOCK_SIZE, &ctx.offset24Ptr, &ctx.offset24End, LIZARD_STREAM_OFFSET24);
if (streamLen == 0) goto _output_error;
#ifdef LIZARD_USE_LOGS
ipos = ip;
streamLen = Lizard_readStream(res&LIZARD_FLAG_FLAGS, &ip, iend, decompFlagsBase, decompFlagsBase + LIZARD_HUF_BLOCK_SIZE, &ctx.flagsPtr, &ctx.flagsEnd, LIZARD_STREAM_FLAGS);
if (streamLen == 0) goto _output_error;
streamLen = (size_t)(ctx.flagsEnd-ctx.flagsPtr);
comprFlagsLen = ((size_t)(ip - ipos) + 3 >= streamLen) ? 0 : (size_t)(ip - ipos);
ipos = ip;
#else
streamLen = Lizard_readStream(res&LIZARD_FLAG_FLAGS, &ip, iend, decompFlagsBase, decompFlagsBase + LIZARD_HUF_BLOCK_SIZE, &ctx.flagsPtr, &ctx.flagsEnd, LIZARD_STREAM_FLAGS);
if (streamLen == 0) goto _output_error;
#endif
streamLen = Lizard_readStream(res&LIZARD_FLAG_LITERALS, &ip, iend, decompLiteralsBase, decompLiteralsBase + LIZARD_HUF_BLOCK_SIZE, &ctx.literalsPtr, &ctx.literalsEnd, LIZARD_STREAM_LITERALS);
if (streamLen == 0) goto _output_error;
#ifdef LIZARD_USE_LOGS
streamLen = (size_t)(ctx.literalsEnd-ctx.literalsPtr);
comprLiteralsLen = ((size_t)(ip - ipos) + 3 >= streamLen) ? 0 : (size_t)(ip - ipos);
total = (size_t)(ip-(ctx.lenEnd-1));
#endif
if (ip > iend) goto _output_error;
LIZARD_LOG_DECOMPRESS("%d: total=%d block=%d flagsLen=%d(HUF=%d) literalsLen=%d(HUF=%d) offset16Len=%d offset24Len=%d lengthsLen=%d \n", (int)(op-(BYTE*)dest) ,(int)(ip-istart), (int)total,
(int)(ctx.flagsEnd-ctx.flagsPtr), (int)comprFlagsLen, (int)(ctx.literalsEnd-ctx.literalsPtr), (int)comprLiteralsLen,
(int)(ctx.offset16End-ctx.offset16Ptr), (int)(ctx.offset24End-ctx.offset24Ptr), (int)(ctx.lenEnd-ctx.lenPtr));
}
ctx.last_off = -LIZARD_INIT_LAST_OFFSET;
params = Lizard_defaultParameters[compressionLevel - LIZARD_MIN_CLEVEL];
if (params.decompressType == Lizard_coderwords_LZ4)
res = Lizard_decompress_LZ4(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel);
else
#ifdef USE_LZ4_ONLY
res = Lizard_decompress_LZ4(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel);
#else
res = Lizard_decompress_LIZv1(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel);
#endif
LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic res=%d inputSize=%d\n", res, (int)(ctx.literalsEnd-ctx.lenEnd));
if (res <= 0) { free(decompLiteralsBase); return res; }
op += res;
outputSize -= res;
if ((partialDecoding) && (op >= oexit)) break;
}
#ifdef LIZARD_STATS
print_stats();
#endif
LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic total=%d\n", (int)(op-(BYTE*)dest));
free(decompLiteralsBase);
return (int)(op-(BYTE*)dest);
_output_error:
LIZARD_LOG_DECOMPRESS("Lizard_decompress_generic ERROR ip=%p iend=%p\n", ip, iend);
free(decompLiteralsBase);
return -1;
}
int Lizard_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
{
return Lizard_decompress_generic(source, dest, compressedSize, maxDecompressedSize, full, 0, noDict, (BYTE*)dest, NULL, 0);
}
int Lizard_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize)
{
return Lizard_decompress_generic(source, dest, compressedSize, maxDecompressedSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0);
}
/*===== streaming decompression functions =====*/
/*
* If you prefer dynamic allocation methods,
* Lizard_createStreamDecode()
* provides a pointer (void*) towards an initialized Lizard_streamDecode_t structure.
*/
Lizard_streamDecode_t* Lizard_createStreamDecode(void)
{
Lizard_streamDecode_t* lizards = (Lizard_streamDecode_t*) ALLOCATOR(1, sizeof(Lizard_streamDecode_t));
return lizards;
}
int Lizard_freeStreamDecode (Lizard_streamDecode_t* Lizard_stream)
{
FREEMEM(Lizard_stream);
return 0;
}
/*!
* Lizard_setStreamDecode() :
* Use this function to instruct where to find the dictionary.
* This function is not necessary if previous data is still available where it was decoded.
* Loading a size of 0 is allowed (same effect as no dictionary).
* Return : 1 if OK, 0 if error
*/
int Lizard_setStreamDecode (Lizard_streamDecode_t* Lizard_streamDecode, const char* dictionary, int dictSize)
{
Lizard_streamDecode_t* lizardsd = (Lizard_streamDecode_t*) Lizard_streamDecode;
lizardsd->prefixSize = (size_t) dictSize;
lizardsd->prefixEnd = (const BYTE*) dictionary + dictSize;
lizardsd->externalDict = NULL;
lizardsd->extDictSize = 0;
return 1;
}
/*
*_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks must still be available at the memory position where they were decoded.
If it's not possible, save the relevant part of decoded data into a safe buffer,
and indicate where it stands using Lizard_setStreamDecode()
*/
int Lizard_decompress_safe_continue (Lizard_streamDecode_t* Lizard_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
{
Lizard_streamDecode_t* lizardsd = (Lizard_streamDecode_t*) Lizard_streamDecode;
int result;
if (lizardsd->prefixEnd == (BYTE*)dest) {
result = Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize,
full, 0, usingExtDict, lizardsd->prefixEnd - lizardsd->prefixSize, lizardsd->externalDict, lizardsd->extDictSize);
if (result <= 0) return result;
lizardsd->prefixSize += result;
lizardsd->prefixEnd += result;
} else {
lizardsd->extDictSize = lizardsd->prefixSize;
lizardsd->externalDict = lizardsd->prefixEnd - lizardsd->extDictSize;
result = Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize,
full, 0, usingExtDict, (BYTE*)dest, lizardsd->externalDict, lizardsd->extDictSize);
if (result <= 0) return result;
lizardsd->prefixSize = result;
lizardsd->prefixEnd = (BYTE*)dest + result;
}
return result;
}
/*
Advanced decoding functions :
*_usingDict() :
These decoding functions work the same as "_continue" ones,
the dictionary must be explicitly provided within parameters
*/
int Lizard_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
if (dictSize==0)
return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, noDict, (BYTE*)dest, NULL, 0);
if (dictStart+dictSize == dest)
{
if (dictSize >= (int)(LIZARD_DICT_SIZE - 1))
return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, withPrefix64k, (BYTE*)dest-LIZARD_DICT_SIZE, NULL, 0);
return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0);
}
return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
}
/* debug function */
int Lizard_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
return Lizard_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
}

152
contrib/lizard/lib/lizard_decompress.h Normal file
View File

@ -0,0 +1,152 @@
/*
Lizard - Fast LZ compression algorithm
Header File
Copyright (C) 2011-2016, Yann Collet
Copyright (C) 2016-2017, Przemyslaw Skibinski <inikep@gmail.com>
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Lizard source repository : https://github.com/inikep/lizard
*/
#ifndef LIZARD_DECOMPRESS_H_2983
#define LIZARD_DECOMPRESS_H_2983
#if defined (__cplusplus)
extern "C" {
#endif
/*^***************************************************************
* Export parameters
*****************************************************************/
/*
* LIZARD_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
*/
#if defined(LIZARD_DLL_EXPORT) && (LIZARD_DLL_EXPORT==1)
# define LIZARDDLIB_API __declspec(dllexport)
#elif defined(LIZARD_DLL_IMPORT) && (LIZARD_DLL_IMPORT==1)
# define LIZARDDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define LIZARDDLIB_API
#endif
/*-************************************
* Simple Functions
**************************************/
/*
Lizard_decompress_safe() :
compressedSize : is the precise full size of the compressed block.
maxDecompressedSize : is the size of destination buffer, which must be already allocated.
return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
If destination buffer is not large enough, decoding will stop and output an error code (<0).
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function is protected against buffer overflow exploits, including malicious data packets.
It never writes outside output buffer, nor reads outside input buffer.
*/
LIZARDDLIB_API int Lizard_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
/*!
Lizard_decompress_safe_partial() :
This function decompress a compressed block of size 'compressedSize' at position 'source'
into destination buffer 'dest' of size 'maxDecompressedSize'.
The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
reducing decompression time.
return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
Always control how many bytes were decoded.
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
*/
LIZARDDLIB_API int Lizard_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
/*-**********************************************
* Streaming Decompression Functions
************************************************/
typedef struct Lizard_streamDecode_s Lizard_streamDecode_t;
/*
* Lizard_streamDecode_t
* information structure to track an Lizard stream.
* init this structure content using Lizard_setStreamDecode or memset() before first use !
*
* In the context of a DLL (liblizard) please prefer usage of construction methods below.
* They are more future proof, in case of a change of Lizard_streamDecode_t size in the future.
* Lizard_createStreamDecode will allocate and initialize an Lizard_streamDecode_t structure
* Lizard_freeStreamDecode releases its memory.
*/
LIZARDDLIB_API Lizard_streamDecode_t* Lizard_createStreamDecode(void);
LIZARDDLIB_API int Lizard_freeStreamDecode (Lizard_streamDecode_t* Lizard_stream);
/*! Lizard_setStreamDecode() :
* Use this function to instruct where to find the dictionary.
* Setting a size of 0 is allowed (same effect as reset).
* @return : 1 if OK, 0 if error
*/
LIZARDDLIB_API int Lizard_setStreamDecode (Lizard_streamDecode_t* Lizard_streamDecode, const char* dictionary, int dictSize);
/*
*_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks *must* remain available at the memory position where they were decoded (up to LIZARD_DICT_SIZE)
In the case of a ring buffers, decoding buffer must be either :
- Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
In which case, the decoding & encoding ring buffer can have any size, including small ones ( < LIZARD_DICT_SIZE).
- Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including small ones ( < LIZARD_DICT_SIZE).
- _At least_ LIZARD_DICT_SIZE + 8 bytes + maxBlockSize.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including larger than decoding buffer.
Whenever these conditions are not possible, save the last LIZARD_DICT_SIZE of decoded data into a safe buffer,
and indicate where it is saved using Lizard_setStreamDecode()
*/
LIZARDDLIB_API int Lizard_decompress_safe_continue (Lizard_streamDecode_t* Lizard_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
/*
Advanced decoding functions :
*_usingDict() :
These decoding functions work the same as
a combination of Lizard_setStreamDecode() followed by Lizard_decompress_x_continue()
They are stand-alone. They don't need nor update an Lizard_streamDecode_t structure.
*/
LIZARDDLIB_API int Lizard_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
#if defined (__cplusplus)
}
#endif
#endif /* LIZARD_DECOMPRESS_H_2983827168210 */

220
contrib/lizard/lib/lizard_decompress_liz.h Normal file
View File

@ -0,0 +1,220 @@
/*
[0_MMMM_LLL] - 16-bit offset, 4-bit match length (4-15+), 3-bit literal length (0-7+)
[1_MMMM_LLL] - last offset, 4-bit match length (0-15+), 3-bit literal length (0-7+)
flag 31 - 24-bit offset, match length (47+), no literal length
flag 0-30 - 24-bit offset, 31 match lengths (16-46), no literal length
*/
/*! Lizard_decompress_LIZv1() :
* This generic decompression function cover all use cases.
* It shall be instantiated several times, using different sets of directives
* Note that it is important this generic function is really inlined,
* in order to remove useless branches during compilation optimization.
*/
FORCE_INLINE int Lizard_decompress_LIZv1(
Lizard_dstream_t* ctx,
BYTE* const dest,
int outputSize, /* this value is the max size of Output Buffer. */
int partialDecoding, /* full, partial */
int targetOutputSize, /* only used if partialDecoding==partial */
int dict, /* noDict, withPrefix64k, usingExtDict */
const BYTE* const lowPrefix, /* == dest if dict == noDict */
const BYTE* const dictStart, /* only if dict==usingExtDict */
const size_t dictSize, /* note : = 0 if noDict */
int compressionLevel
)
{
/* Local Variables */
int inputSize = (int)(ctx->flagsEnd - ctx->flagsPtr);
const BYTE* const blockBase = ctx->flagsPtr;
const BYTE* const iend = ctx->literalsEnd;
BYTE* op = dest;
BYTE* const oend = op + outputSize;
BYTE* cpy = NULL;
BYTE* oexit = op + targetOutputSize;
const BYTE* const lowLimit = lowPrefix - dictSize;
const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
const int checkOffset = (dictSize < (int)(LIZARD_DICT_SIZE));
intptr_t last_off = ctx->last_off;
intptr_t length = 0;
(void)compressionLevel;
/* Special cases */
if (unlikely(outputSize==0)) return ((inputSize==1) && (*ctx->flagsPtr==0)) ? 0 : -1; /* Empty output buffer */
/* Main Loop : decode sequences */
while (ctx->flagsPtr < ctx->flagsEnd) {
unsigned token;
const BYTE* match;
// intptr_t litLength;
if ((partialDecoding) && (op >= oexit)) return (int) (op-dest);
/* get literal length */
token = *ctx->flagsPtr++;
if (token >= 32)
{
if ((length=(token & MAX_SHORT_LITLEN)) == MAX_SHORT_LITLEN) {
if (unlikely(ctx->literalsPtr > iend - 1)) { LIZARD_LOG_DECOMPRESS_LIZv1("1"); goto _output_error; }
length = *ctx->literalsPtr;
if unlikely(length >= 254) {
if (length == 254) {
length = MEM_readLE16(ctx->literalsPtr+1);
ctx->literalsPtr += 2;
} else {
length = MEM_readLE24(ctx->literalsPtr+1);
ctx->literalsPtr += 3;
}
}
length += MAX_SHORT_LITLEN;
ctx->literalsPtr++;
if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LIZv1("2"); goto _output_error; } /* overflow detection */
if (unlikely((size_t)(ctx->literalsPtr+length)<(size_t)(ctx->literalsPtr))) { LIZARD_LOG_DECOMPRESS_LIZv1("3"); goto _output_error; } /* overflow detection */
}
/* copy literals */
cpy = op + length;
if (unlikely(cpy > oend - WILDCOPYLENGTH || ctx->literalsPtr > iend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LIZv1("offset outside buffers\n"); goto _output_error; } /* Error : offset outside buffers */
#if 1
Lizard_wildCopy16(op, ctx->literalsPtr, cpy);
op = cpy;
ctx->literalsPtr += length;
#else
Lizard_copy8(op, ctx->literalsPtr);
Lizard_copy8(op+8, ctx->literalsPtr+8);
if (length > 16)
Lizard_wildCopy16(op + 16, ctx->literalsPtr + 16, cpy);
op = cpy;
ctx->literalsPtr += length;
#endif
/* get offset */
if (unlikely(ctx->offset16Ptr > ctx->offset16End)) { LIZARD_LOG_DECOMPRESS_LIZv1("(ctx->offset16Ptr > ctx->offset16End\n"); goto _output_error; }
#if 1
{ /* branchless */
intptr_t new_off = MEM_readLE16(ctx->offset16Ptr);
uintptr_t not_repCode = (uintptr_t)(token >> ML_RUN_BITS) - 1;
last_off ^= not_repCode & (last_off ^ -new_off);
ctx->offset16Ptr = (BYTE*)((uintptr_t)ctx->offset16Ptr + (not_repCode & 2));
}
#else
if ((token >> ML_RUN_BITS) == 0)
{
last_off = -(intptr_t)MEM_readLE16(ctx->offset16Ptr);
ctx->offset16Ptr += 2;
}
#endif
/* get matchlength */
length = (token >> RUN_BITS_LIZv1) & MAX_SHORT_MATCHLEN;
if (length == MAX_SHORT_MATCHLEN) {
if (unlikely(ctx->literalsPtr > iend - 1)) { LIZARD_LOG_DECOMPRESS_LIZv1("6"); goto _output_error; }
length = *ctx->literalsPtr;
if unlikely(length >= 254) {
if (length == 254) {
length = MEM_readLE16(ctx->literalsPtr+1);
ctx->literalsPtr += 2;
} else {
length = MEM_readLE24(ctx->literalsPtr+1);
ctx->literalsPtr += 3;
}
}
length += MAX_SHORT_MATCHLEN;
ctx->literalsPtr++;
if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LIZv1("7"); goto _output_error; } /* overflow detection */
}
DECOMPLOG_CODEWORDS_LIZv1("T32+ literal=%u match=%u offset=%d ipos=%d opos=%d\n", (U32)litLength, (U32)length, (int)-last_off, (U32)(ctx->flagsPtr-blockBase), (U32)(op-dest));
}
else
if (token < LIZARD_LAST_LONG_OFF)
{
if (unlikely(ctx->offset24Ptr > ctx->offset24End - 3)) { LIZARD_LOG_DECOMPRESS_LIZv1("8"); goto _output_error; }
length = token + MM_LONGOFF;
last_off = -(intptr_t)MEM_readLE24(ctx->offset24Ptr);
ctx->offset24Ptr += 3;
DECOMPLOG_CODEWORDS_LIZv1("T0-30 literal=%u match=%u offset=%d\n", 0, (U32)length, (int)-last_off);
}
else
{
if (unlikely(ctx->literalsPtr > iend - 1)) { LIZARD_LOG_DECOMPRESS_LIZv1("9"); goto _output_error; }
length = *ctx->literalsPtr;
if unlikely(length >= 254) {
if (length == 254) {
length = MEM_readLE16(ctx->literalsPtr+1);
ctx->literalsPtr += 2;
} else {
length = MEM_readLE24(ctx->literalsPtr+1);
ctx->literalsPtr += 3;
}
}
ctx->literalsPtr++;
length += LIZARD_LAST_LONG_OFF + MM_LONGOFF;
if (unlikely(ctx->offset24Ptr > ctx->offset24End - 3)) { LIZARD_LOG_DECOMPRESS_LIZv1("10"); goto _output_error; }
last_off = -(intptr_t)MEM_readLE24(ctx->offset24Ptr);
ctx->offset24Ptr += 3;
}
match = op + last_off;
if ((checkOffset) && ((unlikely((uintptr_t)(-last_off) > (uintptr_t)op) || (match < lowLimit)))) { LIZARD_LOG_DECOMPRESS_LIZv1("lowPrefix[%p]-dictSize[%d]=lowLimit[%p] match[%p]=op[%p]-last_off[%d]\n", lowPrefix, (int)dictSize, lowLimit, match, op, (int)last_off); goto _output_error; } /* Error : offset outside buffers */
/* check external dictionary */
if ((dict==usingExtDict) && (match < lowPrefix)) {
if (unlikely(op + length > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LIZv1("12"); goto _output_error; } /* doesn't respect parsing restriction */
if (length <= (intptr_t)(lowPrefix - match)) {
/* match can be copied as a single segment from external dictionary */
memmove(op, dictEnd - (lowPrefix-match), length);
op += length;
} else {
/* match encompass external dictionary and current block */
size_t const copySize = (size_t)(lowPrefix-match);
size_t const restSize = length - copySize;
memcpy(op, dictEnd - copySize, copySize);
op += copySize;
if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */
BYTE* const endOfMatch = op + restSize;
const BYTE* copyFrom = lowPrefix;
while (op < endOfMatch) *op++ = *copyFrom++;
} else {
memcpy(op, lowPrefix, restSize);
op += restSize;
} }
continue;
}
/* copy match within block */
cpy = op + length;
if (unlikely(cpy > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LIZv1("13match=%p lowLimit=%p\n", match, lowLimit); goto _output_error; } /* Error : offset outside buffers */
Lizard_copy8(op, match);
Lizard_copy8(op+8, match+8);
if (length > 16)
Lizard_wildCopy16(op + 16, match + 16, cpy);
op = cpy;
}
/* last literals */
length = ctx->literalsEnd - ctx->literalsPtr;
cpy = op + length;
if ((length < 0) || (ctx->literalsPtr+length != iend) || (cpy > oend)) { LIZARD_LOG_DECOMPRESS_LIZv1("14"); goto _output_error; } /* Error : input must be consumed */
memcpy(op, ctx->literalsPtr, length);
ctx->literalsPtr += length;
op += length;
/* end of decoding */
ctx->last_off = last_off;
return (int) (op-dest); /* Nb of output bytes decoded */
/* Overflow error detected */
_output_error:
LIZARD_LOG_DECOMPRESS_LIZv1("_output_error=%d ctx->flagsPtr=%p blockBase=%p\n", (int) (-(ctx->flagsPtr-blockBase))-1, ctx->flagsPtr, blockBase);
LIZARD_LOG_DECOMPRESS_LIZv1("cpy=%p oend=%p ctx->literalsPtr+length[%d]=%p iend=%p\n", cpy, oend, (int)length, ctx->literalsPtr+length, iend);
return (int) (-(ctx->flagsPtr-blockBase))-1;
}

163
contrib/lizard/lib/lizard_decompress_lz4.h Normal file
View File

@ -0,0 +1,163 @@
/*! Lizard_decompress_LZ4() :
* This generic decompression function cover all use cases.
* It shall be instantiated several times, using different sets of directives
* Note that it is important this generic function is really inlined,
* in order to remove useless branches during compilation optimization.
*/
FORCE_INLINE int Lizard_decompress_LZ4(
Lizard_dstream_t* ctx,
BYTE* const dest,
int outputSize, /* this value is the max size of Output Buffer. */
int partialDecoding, /* full, partial */
int targetOutputSize, /* only used if partialDecoding==partial */
int dict, /* noDict, withPrefix64k, usingExtDict */
const BYTE* const lowPrefix, /* == dest if dict == noDict */
const BYTE* const dictStart, /* only if dict==usingExtDict */
const size_t dictSize, /* note : = 0 if noDict */
int compressionLevel
)
{
/* Local Variables */
int inputSize = (int)(ctx->flagsEnd - ctx->flagsPtr);
const BYTE* const blockBase = ctx->flagsPtr;
const BYTE* const iend = ctx->literalsEnd;
BYTE* op = dest;
BYTE* const oend = op + outputSize;
BYTE* cpy = NULL;
BYTE* oexit = op + targetOutputSize;
const BYTE* const lowLimit = lowPrefix - dictSize;
const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
const int checkOffset = (dictSize < (int)(LIZARD_DICT_SIZE));
intptr_t length = 0;
(void)compressionLevel;
/* Special cases */
if (unlikely(outputSize==0)) return ((inputSize==1) && (*ctx->flagsPtr==0)) ? 0 : -1; /* Empty output buffer */
/* Main Loop : decode sequences */
while (ctx->flagsPtr < ctx->flagsEnd) {
unsigned token;
const BYTE* match;
size_t offset;
/* get literal length */
token = *ctx->flagsPtr++;
if ((length=(token & RUN_MASK_LZ4)) == RUN_MASK_LZ4) {
if (unlikely(ctx->literalsPtr > iend - 5)) { LIZARD_LOG_DECOMPRESS_LZ4("0"); goto _output_error; }
length = *ctx->literalsPtr;
if unlikely(length >= 254) {
if (length == 254) {
length = MEM_readLE16(ctx->literalsPtr+1);
ctx->literalsPtr += 2;
} else {
length = MEM_readLE24(ctx->literalsPtr+1);
ctx->literalsPtr += 3;
}
}
length += RUN_MASK_LZ4;
ctx->literalsPtr++;
if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LZ4("1"); goto _output_error; } /* overflow detection */
if (unlikely((size_t)(ctx->literalsPtr+length)<(size_t)(ctx->literalsPtr))) { LIZARD_LOG_DECOMPRESS_LZ4("2"); goto _output_error; } /* overflow detection */
}
/* copy literals */
cpy = op + length;
if (unlikely(cpy > oend - WILDCOPYLENGTH || ctx->literalsPtr + length > iend - (2 + WILDCOPYLENGTH))) { LIZARD_LOG_DECOMPRESS_LZ4("offset outside buffers\n"); goto _output_error; } /* Error : offset outside buffers */
#if 1
Lizard_wildCopy16(op, ctx->literalsPtr, cpy);
op = cpy;
ctx->literalsPtr += length;
#else
Lizard_copy8(op, ctx->literalsPtr);
Lizard_copy8(op+8, ctx->literalsPtr+8);
if (length > 16)
Lizard_wildCopy16(op + 16, ctx->literalsPtr + 16, cpy);
op = cpy;
ctx->literalsPtr += length;
#endif
if ((partialDecoding) && (op >= oexit)) return (int) (op-dest);
/* get offset */
offset = MEM_readLE16(ctx->literalsPtr);
ctx->literalsPtr += 2;
match = op - offset;
if ((checkOffset) && (unlikely(match < lowLimit))) { LIZARD_LOG_DECOMPRESS_LZ4("lowPrefix[%p]-dictSize[%d]=lowLimit[%p] match[%p]=op[%p]-offset[%d]\n", lowPrefix, (int)dictSize, lowLimit, match, op, (int)offset); goto _output_error; } /* Error : offset outside buffers */
/* get matchlength */
length = token >> RUN_BITS_LZ4;
if (length == ML_MASK_LZ4) {
if (unlikely(ctx->literalsPtr > iend - 5)) { LIZARD_LOG_DECOMPRESS_LZ4("4"); goto _output_error; }
length = *ctx->literalsPtr;
if unlikely(length >= 254) {
if (length == 254) {
length = MEM_readLE16(ctx->literalsPtr+1);
ctx->literalsPtr += 2;
} else {
length = MEM_readLE24(ctx->literalsPtr+1);
ctx->literalsPtr += 3;
}
}
length += ML_MASK_LZ4;
ctx->literalsPtr++;
if (unlikely((size_t)(op+length)<(size_t)(op))) { LIZARD_LOG_DECOMPRESS_LZ4("5"); goto _output_error; } /* overflow detection */
}
length += MINMATCH;
/* check external dictionary */
if ((dict==usingExtDict) && (match < lowPrefix)) {
if (unlikely(op + length > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LZ4("6"); goto _output_error; } /* doesn't respect parsing restriction */
if (length <= (intptr_t)(lowPrefix - match)) {
/* match can be copied as a single segment from external dictionary */
memmove(op, dictEnd - (lowPrefix-match), length);
op += length;
} else {
/* match encompass external dictionary and current block */
size_t const copySize = (size_t)(lowPrefix-match);
size_t const restSize = length - copySize;
memcpy(op, dictEnd - copySize, copySize);
op += copySize;
if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */
BYTE* const endOfMatch = op + restSize;
const BYTE* copyFrom = lowPrefix;
while (op < endOfMatch) *op++ = *copyFrom++;
} else {
memcpy(op, lowPrefix, restSize);
op += restSize;
} }
continue;
}
/* copy match within block */
cpy = op + length;
if (unlikely(cpy > oend - WILDCOPYLENGTH)) { LIZARD_LOG_DECOMPRESS_LZ4("1match=%p lowLimit=%p\n", match, lowLimit); goto _output_error; } /* Error : offset outside buffers */
Lizard_copy8(op, match);
Lizard_copy8(op+8, match+8);
if (length > 16)
Lizard_wildCopy16(op + 16, match + 16, cpy);
op = cpy;
if ((partialDecoding) && (op >= oexit)) return (int) (op-dest);
}
/* last literals */
length = ctx->literalsEnd - ctx->literalsPtr;
cpy = op + length;
if ((length < 0) || (ctx->literalsPtr+length != iend) || (cpy > oend)) { LIZARD_LOG_DECOMPRESS_LZ4("9"); goto _output_error; } /* Error : input must be consumed */
memcpy(op, ctx->literalsPtr, length);
ctx->literalsPtr += length;
op += length;
/* end of decoding */
return (int) (op-dest); /* Nb of output bytes decoded */
/* Overflow error detected */
_output_error:
LIZARD_LOG_DECOMPRESS_LZ4("_output_error=%d ctx->flagsPtr=%p blockBase=%p\n", (int) (-(ctx->flagsPtr-blockBase))-1, ctx->flagsPtr, blockBase);
LIZARD_LOG_DECOMPRESS_LZ4("cpy=%p oend=%p ctx->literalsPtr+length[%d]=%p iend=%p\n", cpy, oend, (int)length, ctx->literalsPtr+length, iend);
return (int) (-(ctx->flagsPtr-blockBase))-1;
}

1362
contrib/lizard/lib/lizard_frame.c Normal file
View File

File diff suppressed because it is too large Load Diff

303
contrib/lizard/lib/lizard_frame.h Normal file
View File

@ -0,0 +1,303 @@
/*
Lizard auto-framing library
Header File
Copyright (C) 2011-2015, Yann Collet
Copyright (C) 2016-2017, Przemyslaw Skibinski
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Lizard source repository : https://github.com/inikep/lizard
*/
/* LizardF is a stand-alone API to create Lizard-compressed frames
* conformant with specification v1.5.1.
* All related operations, including memory management, are handled internally by the library.
* You don't need lizard_compress.h when using lizard_frame.h.
* */
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/*-************************************
* Includes
**************************************/
#include <stddef.h> /* size_t */
/*-************************************
* Error management
**************************************/
typedef size_t LizardF_errorCode_t;
unsigned LizardF_isError(LizardF_errorCode_t code);
const char* LizardF_getErrorName(LizardF_errorCode_t code); /* return error code string; useful for debugging */
/*-************************************
* Frame compression types
**************************************/
//#define LIZARDF_DISABLE_OBSOLETE_ENUMS
#ifndef LIZARDF_DISABLE_OBSOLETE_ENUMS
# define LIZARDF_OBSOLETE_ENUM(x) ,x
#else
# define LIZARDF_OBSOLETE_ENUM(x)
#endif
typedef enum {
LizardF_default=0,
LizardF_max128KB=1,
LizardF_max256KB=2,
LizardF_max1MB=3,
LizardF_max4MB=4,
LizardF_max16MB=5,
LizardF_max64MB=6,
LizardF_max256MB=7
} LizardF_blockSizeID_t;
typedef enum {
LizardF_blockLinked=0,
LizardF_blockIndependent
LIZARDF_OBSOLETE_ENUM(blockLinked = LizardF_blockLinked)
LIZARDF_OBSOLETE_ENUM(blockIndependent = LizardF_blockIndependent)
} LizardF_blockMode_t;
typedef enum {
LizardF_noContentChecksum=0,
LizardF_contentChecksumEnabled
LIZARDF_OBSOLETE_ENUM(noContentChecksum = LizardF_noContentChecksum)
LIZARDF_OBSOLETE_ENUM(contentChecksumEnabled = LizardF_contentChecksumEnabled)
} LizardF_contentChecksum_t;
typedef enum {
LizardF_frame=0,
LizardF_skippableFrame
LIZARDF_OBSOLETE_ENUM(skippableFrame = LizardF_skippableFrame)
} LizardF_frameType_t;
#ifndef LIZARDF_DISABLE_OBSOLETE_ENUMS
typedef LizardF_blockSizeID_t blockSizeID_t;
typedef LizardF_blockMode_t blockMode_t;
typedef LizardF_frameType_t frameType_t;
typedef LizardF_contentChecksum_t contentChecksum_t;
#endif
typedef struct {
LizardF_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
LizardF_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
LizardF_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */
LizardF_frameType_t frameType; /* LizardF_frame, skippableFrame ; 0 == default */
unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */
unsigned reserved[2]; /* must be zero for forward compatibility */
} LizardF_frameInfo_t;
typedef struct {
LizardF_frameInfo_t frameInfo;
int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */
unsigned reserved[4]; /* must be zero for forward compatibility */
} LizardF_preferences_t;
/*-*********************************
* Simple compression function
***********************************/
size_t LizardF_compressFrameBound(size_t srcSize, const LizardF_preferences_t* preferencesPtr);
/*!LizardF_compressFrame() :
* Compress an entire srcBuffer into a valid Lizard frame, as defined by specification v1.5.1
* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
* You can get the minimum value of dstMaxSize by using LizardF_compressFrameBound()
* If this condition is not respected, LizardF_compressFrame() will fail (result is an errorCode)
* The LizardF_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
* The result of the function is the number of bytes written into dstBuffer.
* The function outputs an error code if it fails (can be tested using LizardF_isError())
*/
size_t LizardF_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_preferences_t* preferencesPtr);
/*-***********************************
* Advanced compression functions
*************************************/
typedef struct LizardF_cctx_s* LizardF_compressionContext_t; /* must be aligned on 8-bytes */
typedef struct {
unsigned stableSrc; /* 1 == src content will remain available on future calls to LizardF_compress(); avoid saving src content within tmp buffer as future dictionary */
unsigned reserved[3];
} LizardF_compressOptions_t;
/* Resource Management */
#define LIZARDF_VERSION 100
LizardF_errorCode_t LizardF_createCompressionContext(LizardF_compressionContext_t* cctxPtr, unsigned version);
LizardF_errorCode_t LizardF_freeCompressionContext(LizardF_compressionContext_t cctx);
/* LizardF_createCompressionContext() :
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
* This is achieved using LizardF_createCompressionContext(), which takes as argument a version and an LizardF_preferences_t structure.
* The version provided MUST be LIZARDF_VERSION. It is intended to track potential version differences between different binaries.
* The function will provide a pointer to a fully allocated LizardF_compressionContext_t object.
* If the result LizardF_errorCode_t is not zero, there was an error during context creation.
* Object can release its memory using LizardF_freeCompressionContext();
*/
/* Compression */
size_t LizardF_compressBegin(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LizardF_preferences_t* prefsPtr);
/* LizardF_compressBegin() :
* will write the frame header into dstBuffer.
* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes.
* The LizardF_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default.
* The result of the function is the number of bytes written into dstBuffer for the header
* or an error code (can be tested using LizardF_isError())
*/
size_t LizardF_compressBound(size_t srcSize, const LizardF_preferences_t* prefsPtr);
/* LizardF_compressBound() :
* Provides the minimum size of Dst buffer given srcSize to handle worst case situations.
* Different preferences can produce different results.
* prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case.
* This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled)
*/
size_t LizardF_compressUpdate(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LizardF_compressOptions_t* cOptPtr);
/* LizardF_compressUpdate()
* LizardF_compressUpdate() can be called repetitively to compress as much data as necessary.
* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
* You can get the minimum value of dstMaxSize by using LizardF_compressBound().
* If this condition is not respected, LizardF_compress() will fail (result is an errorCode).
* LizardF_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs.
* The LizardF_compressOptions_t structure is optional : you can provide NULL as argument.
* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered.
* The function outputs an error code if it fails (can be tested using LizardF_isError())
*/
size_t LizardF_flush(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* cOptPtr);
/* LizardF_flush()
* Should you need to generate compressed data immediately, without waiting for the current block to be filled,
* you can call Lizard_flush(), which will immediately compress any remaining data buffered within cctx.
* Note that dstMaxSize must be large enough to ensure the operation will be successful.
* LizardF_compressOptions_t structure is optional : you can provide NULL as argument.
* The result of the function is the number of bytes written into dstBuffer
* (it can be zero, this means there was no data left within cctx)
* The function outputs an error code if it fails (can be tested using LizardF_isError())
*/
size_t LizardF_compressEnd(LizardF_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LizardF_compressOptions_t* cOptPtr);
/* LizardF_compressEnd()
* When you want to properly finish the compressed frame, just call LizardF_compressEnd().
* It will flush whatever data remained within compressionContext (like Lizard_flush())
* but also properly finalize the frame, with an endMark and a checksum.
* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
* The function outputs an error code if it fails (can be tested using LizardF_isError())
* The LizardF_compressOptions_t structure is optional : you can provide NULL as argument.
* A successful call to LizardF_compressEnd() makes cctx available again for next compression task.
*/
/*-*********************************
* Decompression functions
***********************************/
typedef struct LizardF_dctx_s* LizardF_decompressionContext_t; /* must be aligned on 8-bytes */
typedef struct {
unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
unsigned reserved[3];
} LizardF_decompressOptions_t;
/* Resource management */
/*!LizardF_createDecompressionContext() :
* Create an LizardF_decompressionContext_t object, which will be used to track all decompression operations.
* The version provided MUST be LIZARDF_VERSION. It is intended to track potential breaking differences between different versions.
* The function will provide a pointer to a fully allocated and initialized LizardF_decompressionContext_t object.
* The result is an errorCode, which can be tested using LizardF_isError().
* dctx memory can be released using LizardF_freeDecompressionContext();
* The result of LizardF_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
* That is, it should be == 0 if decompression has been completed fully and correctly.
*/
LizardF_errorCode_t LizardF_createDecompressionContext(LizardF_decompressionContext_t* dctxPtr, unsigned version);
LizardF_errorCode_t LizardF_freeDecompressionContext(LizardF_decompressionContext_t dctx);
/*====== Decompression ======*/
/*!LizardF_getFrameInfo() :
* This function decodes frame header information (such as max blockSize, frame checksum, etc.).
* Its usage is optional. The objective is to extract frame header information, typically for allocation purposes.
* A header size is variable and can be from 7 to 15 bytes. It's also possible to input more bytes than that.
* The number of bytes read from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
* (note that LizardF_getFrameInfo() can also be used anytime *after* starting decompression, in this case 0 input byte is enough)
* Frame header info is *copied into* an already allocated LizardF_frameInfo_t structure.
* The function result is an hint about how many srcSize bytes LizardF_decompress() expects for next call,
* or an error code which can be tested using LizardF_isError()
* (typically, when there is not enough src bytes to fully decode the frame header)
* Decompression is expected to resume from where it stopped (srcBuffer + *srcSizePtr)
*/
size_t LizardF_getFrameInfo(LizardF_decompressionContext_t dctx,
LizardF_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
/*!LizardF_decompress() :
* Call this function repetitively to regenerate data compressed within srcBuffer.
* The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr.
*
* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
*
* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
* If number of bytes read is < number of bytes provided, then decompression operation is not completed.
* It typically happens when dstBuffer is not large enough to contain all decoded data.
* LizardF_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr)
* The function will check this condition, and refuse to continue if it is not respected.
*
* `dstBuffer` is expected to be flushed between each call to the function, its content will be overwritten.
* `dst` arguments can be changed at will at each consecutive call to the function.
*
* The function result is an hint of how many `srcSize` bytes LizardF_decompress() expects for next call.
* Schematically, it's the size of the current (or remaining) compressed block + header of next block.
* Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
* This is just a hint though, it's always possible to provide any srcSize.
* When a frame is fully decoded, the function result will be 0 (no more data expected).
* If decompression failed, function result is an error code, which can be tested using LizardF_isError().
*
* After a frame is fully decoded, dctx can be used again to decompress another frame.
*/
size_t LizardF_decompress(LizardF_decompressionContext_t dctx,
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
const LizardF_decompressOptions_t* dOptPtr);
#if defined (__cplusplus)
}
#endif

81
contrib/lizard/lib/lizard_frame_static.h Normal file
View File

@ -0,0 +1,81 @@
/*
Lizard auto-framing library
Header File for static linking only
Copyright (C) 2011-2015, Yann Collet.
Copyright (C) 2016-2017, Przemyslaw Skibinski
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Lizard source repository : https://github.com/inikep/lizard
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/* lizard_frame_static.h should be used solely in the context of static linking.
* It contains definitions which may still change overtime.
* Never use it in the context of DLL linking.
* */
/**************************************
* Includes
**************************************/
#include "lizard_frame.h"
/**************************************
* Error management
* ************************************/
#define LIZARDF_LIST_ERRORS(ITEM) \
ITEM(OK_NoError) ITEM(ERROR_GENERIC) \
ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \
ITEM(ERROR_compressionLevel_invalid) \
ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \
ITEM(ERROR_allocation_failed) \
ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \
ITEM(ERROR_srcPtr_wrong) \
ITEM(ERROR_decompressionFailed) \
ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \
ITEM(ERROR_maxCode)
//#define LIZARDF_DISABLE_OLD_ENUMS
#ifndef LIZARDF_DISABLE_OLD_ENUMS
#define LIZARDF_GENERATE_ENUM(ENUM) LizardF_##ENUM, ENUM = LizardF_##ENUM,
#else
#define LIZARDF_GENERATE_ENUM(ENUM) LizardF_##ENUM,
#endif
typedef enum { LIZARDF_LIST_ERRORS(LIZARDF_GENERATE_ENUM) } LizardF_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */
#if defined (__cplusplus)
}
#endif

196
contrib/lizard/lib/lizard_parser_fast.h Normal file
View File

@ -0,0 +1,196 @@
#define LIZARD_FAST_MIN_OFFSET 8
#define LIZARD_FAST_LONGOFF_MM 0 /* not used with offsets > 1<<16 */
/**************************************
* Hash Functions
**************************************/
static size_t Lizard_hashPosition(const void* p)
{
if (MEM_64bits())
return Lizard_hash5Ptr(p, LIZARD_HASHLOG_LZ4);
return Lizard_hash4Ptr(p, LIZARD_HASHLOG_LZ4);
}
static void Lizard_putPositionOnHash(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase)
{
hashTable[h] = (U32)(p-srcBase);
}
static void Lizard_putPosition(const BYTE* p, U32* hashTable, const BYTE* srcBase)
{
size_t const h = Lizard_hashPosition(p);
Lizard_putPositionOnHash(p, h, hashTable, srcBase);
}
static U32 Lizard_getPositionOnHash(size_t h, U32* hashTable)
{
return hashTable[h];
}
static U32 Lizard_getPosition(const BYTE* p, U32* hashTable)
{
size_t const h = Lizard_hashPosition(p);
return Lizard_getPositionOnHash(h, hashTable);
}
static const U32 Lizard_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */
static const U32 Lizard_minLength = (MFLIMIT+1);
FORCE_INLINE int Lizard_compress_fast(
Lizard_stream_t* const ctx,
const BYTE* ip,
const BYTE* const iend)
{
const U32 acceleration = 1;
const BYTE* base = ctx->base;
const U32 dictLimit = ctx->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const BYTE* const dictBase = ctx->dictBase;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = iend - LASTLITERALS;
const BYTE* anchor = ip;
size_t forwardH, matchIndex;
const U32 maxDistance = (1 << ctx->params.windowLog) - 1;
const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip - base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance;
/* Init conditions */
if ((U32)(iend-ip) > (U32)LIZARD_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */
if ((U32)(iend-ip) < Lizard_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* First Byte */
Lizard_putPosition(ip, ctx->hashTable, base);
ip++; forwardH = Lizard_hashPosition(ip);
/* Main Loop */
for ( ; ; ) {
const BYTE* match;
size_t matchLength;
/* Find a match */
{ const BYTE* forwardIp = ip;
unsigned step = 1;
unsigned searchMatchNb = acceleration << Lizard_skipTrigger;
while (1) {
size_t const h = forwardH;
ip = forwardIp;
forwardIp += step;
step = (searchMatchNb++ >> Lizard_skipTrigger);
if (unlikely(forwardIp > mflimit)) goto _last_literals;
matchIndex = Lizard_getPositionOnHash(h, ctx->hashTable);
forwardH = Lizard_hashPosition(forwardIp);
Lizard_putPositionOnHash(ip, h, ctx->hashTable, base);
if ((matchIndex < lowLimit) || (matchIndex >= (U32)(ip - base)) || (base + matchIndex + maxDistance < ip)) continue;
if (matchIndex >= dictLimit) {
match = base + matchIndex;
#if LIZARD_FAST_MIN_OFFSET > 0
if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET)
#endif
if (MEM_read32(match) == MEM_read32(ip))
{
int back = 0;
matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit);
while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--;
matchLength -= back;
#if LIZARD_FAST_LONGOFF_MM > 0
if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET))
#endif
{
ip += back;
match += back;
break;
}
}
} else {
match = dictBase + matchIndex;
#if LIZARD_FAST_MIN_OFFSET > 0
if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET)
#endif
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance;
int back = 0;
matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr);
while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--;
matchLength -= back;
match = base + matchIndex + back;
#if LIZARD_FAST_LONGOFF_MM > 0
if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET))
#endif
{
ip += back;
break;
}
}
}
} // while (1)
}
_next_match:
if (Lizard_encodeSequence_LZ4(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error;
/* Test end of chunk */
if (ip > mflimit) break;
/* Fill table */
Lizard_putPosition(ip-2, ctx->hashTable, base);
/* Test next position */
matchIndex = Lizard_getPosition(ip, ctx->hashTable);
Lizard_putPosition(ip, ctx->hashTable, base);
if ((matchIndex >= lowLimit) && (matchIndex < (U32)(ip - base)) && (base + matchIndex + maxDistance >= ip))
{
if (matchIndex >= dictLimit) {
match = base + matchIndex;
#if LIZARD_FAST_MIN_OFFSET > 0
if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET)
#endif
if (MEM_read32(match) == MEM_read32(ip))
{
matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit);
#if LIZARD_FAST_LONGOFF_MM > 0
if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET))
#endif
goto _next_match;
}
} else {
match = dictBase + matchIndex;
#if LIZARD_FAST_MIN_OFFSET > 0
if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET)
#endif
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr);
match = base + matchIndex;
#if LIZARD_FAST_LONGOFF_MM > 0
if ((matchLength >= LIZARD_FAST_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET))
#endif
goto _next_match;
}
}
}
/* Prepare next loop */
forwardH = Lizard_hashPosition(++ip);
}
_last_literals:
/* Encode Last Literals */
ip = iend;
if (Lizard_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error;
/* End */
return 1;
_output_error:
return 0;
}

175
contrib/lizard/lib/lizard_parser_fastbig.h Normal file
View File

@ -0,0 +1,175 @@
#define LIZARD_FASTBIG_LONGOFF_MM MM_LONGOFF
/**************************************
* Hash Functions
**************************************/
static size_t Lizard_hashPositionHLog(const void* p, int hashLog)
{
if (MEM_64bits())
return Lizard_hash5Ptr(p, hashLog);
return Lizard_hash4Ptr(p, hashLog);
}
static void Lizard_putPositionOnHashHLog(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase)
{
hashTable[h] = (U32)(p-srcBase);
}
static void Lizard_putPositionHLog(const BYTE* p, U32* hashTable, const BYTE* srcBase, int hashLog)
{
size_t const h = Lizard_hashPositionHLog(p, hashLog);
Lizard_putPositionOnHashHLog(p, h, hashTable, srcBase);
}
static U32 Lizard_getPositionOnHashHLog(size_t h, U32* hashTable)
{
return hashTable[h];
}
static U32 Lizard_getPositionHLog(const BYTE* p, U32* hashTable, int hashLog)
{
size_t const h = Lizard_hashPositionHLog(p, hashLog);
return Lizard_getPositionOnHashHLog(h, hashTable);
}
FORCE_INLINE int Lizard_compress_fastBig(
Lizard_stream_t* const ctx,
const BYTE* ip,
const BYTE* const iend)
{
const U32 acceleration = 1;
const BYTE* base = ctx->base;
const U32 dictLimit = ctx->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const BYTE* const dictBase = ctx->dictBase;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = iend - LASTLITERALS;
const BYTE* anchor = ip;
size_t forwardH, matchIndex;
const int hashLog = ctx->params.hashLog;
const U32 maxDistance = (1 << ctx->params.windowLog) - 1;
const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip - base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance;
/* Init conditions */
if ((U32)(iend-ip) > (U32)LIZARD_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */
if ((U32)(iend-ip) < Lizard_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* First Byte */
Lizard_putPositionHLog(ip, ctx->hashTable, base, hashLog);
ip++; forwardH = Lizard_hashPositionHLog(ip, hashLog);
/* Main Loop */
for ( ; ; ) {
const BYTE* match;
size_t matchLength;
/* Find a match */
{ const BYTE* forwardIp = ip;
unsigned step = 1;
unsigned searchMatchNb = acceleration << Lizard_skipTrigger;
while (1) {
size_t const h = forwardH;
ip = forwardIp;
forwardIp += step;
step = (searchMatchNb++ >> Lizard_skipTrigger);
if (unlikely(forwardIp > mflimit)) goto _last_literals;
matchIndex = Lizard_getPositionOnHashHLog(h, ctx->hashTable);
forwardH = Lizard_hashPositionHLog(forwardIp, hashLog);
Lizard_putPositionOnHashHLog(ip, h, ctx->hashTable, base);
if ((matchIndex < lowLimit) || (matchIndex >= (U32)(ip - base)) || (base + matchIndex + maxDistance < ip)) continue;
if (matchIndex >= dictLimit) {
match = base + matchIndex;
if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET)
if (MEM_read32(match) == MEM_read32(ip))
{
int back = 0;
matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit);
while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--;
matchLength -= back;
if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET))
{
ip += back;
match += back;
break;
}
}
} else {
match = dictBase + matchIndex;
if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET)
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance;
int back = 0;
matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr);
while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--;
matchLength -= back;
match = base + matchIndex + back;
if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET))
{
ip += back;
break;
}
}
}
} // while (1)
}
_next_match:
if (Lizard_encodeSequence_LIZv1(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error;
/* Test end of chunk */
if (ip > mflimit) break;
/* Fill table */
Lizard_putPositionHLog(ip-2, ctx->hashTable, base, hashLog);
/* Test next position */
matchIndex = Lizard_getPositionHLog(ip, ctx->hashTable, hashLog);
Lizard_putPositionHLog(ip, ctx->hashTable, base, hashLog);
if ((matchIndex >= lowLimit) && (matchIndex < (U32)(ip - base)) && (base + matchIndex + maxDistance >= ip))
{
if (matchIndex >= dictLimit) {
match = base + matchIndex;
if ((U32)(ip - match) >= LIZARD_FAST_MIN_OFFSET)
if (MEM_read32(match) == MEM_read32(ip))
{
matchLength = Lizard_count(ip+MINMATCH, match+MINMATCH, matchlimit);
if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET))
goto _next_match;
}
} else {
match = dictBase + matchIndex;
if ((U32)(ip - (base + matchIndex)) >= LIZARD_FAST_MIN_OFFSET)
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
matchLength = Lizard_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr);
match = base + matchIndex;
if ((matchLength >= LIZARD_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LIZARD_MAX_16BIT_OFFSET))
goto _next_match;
}
}
}
/* Prepare next loop */
forwardH = Lizard_hashPositionHLog(++ip, hashLog);
}
_last_literals:
/* Encode Last Literals */
ip = iend;
if (Lizard_encodeLastLiterals_LIZv1(ctx, &ip, &anchor)) goto _output_error;
/* End */
return 1;
_output_error:
return 0;
}

Some files were not shown because too many files have changed in this diff Show More