mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-25 00:52:02 +00:00
Updated zstd to 0.6.1 [#METR-21075].
This commit is contained in:
Alexey Milovidov
parent
4ad5fabbe7
commit
9594acf860
@ -1 +1,12 @@
|
||||
add_library (zstd include/zstd/zstd_compress.c include/zstd/zstd_decompress.c include/zstd/huff0.c include/zstd/fse.c)
|
||||
add_library (zstd
|
||||
include/zstd/entropy_common.c
|
||||
include/zstd/fse_compress.c
|
||||
include/zstd/fse_decompress.c
|
||||
include/zstd/huf_compress.c
|
||||
include/zstd/huf_decompress.c
|
||||
include/zstd/zbuff_compress.c
|
||||
include/zstd/zbuff_decompress.c
|
||||
include/zstd/zstd_common.c
|
||||
include/zstd/zstd_compress.c
|
||||
include/zstd/zstd_decompress.c
|
||||
)
|
||||
|
||||
@ -1 +1 @@
|
||||
https://github.com/Cyan4973/zstd/tree/v0.6.0
|
||||
https://github.com/Cyan4973/zstd/tree/v0.6.1
|
||||
|
||||
@ -84,7 +84,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
|
||||
|
||||
/* Start with initCStream, providing the size of buffer to write into.
|
||||
* bitStream will never write outside of this buffer.
|
||||
* `dstCapacity` must be >= sizeof(size_t), otherwise @return will be an error code.
|
||||
* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
|
||||
*
|
||||
* bits are first added to a local register.
|
||||
* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
|
||||
@ -128,7 +128,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
||||
* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
|
||||
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
|
||||
* Otherwise, it can be less than that, so proceed accordingly.
|
||||
* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
|
||||
*/
|
||||
@ -263,9 +263,9 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
{
|
||||
if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
|
||||
|
||||
if (srcSize >= sizeof(size_t)) { /* normal case */
|
||||
if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
|
||||
bitD->start = (const char*)srcBuffer;
|
||||
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(size_t);
|
||||
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */
|
||||
@ -276,53 +276,58 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
bitD->bitContainer = *(const BYTE*)(bitD->start);
|
||||
switch(srcSize)
|
||||
{
|
||||
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
||||
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
||||
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
||||
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
||||
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
||||
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
||||
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
|
||||
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
|
||||
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
|
||||
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
|
||||
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
|
||||
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
|
||||
default:;
|
||||
}
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */
|
||||
bitD->bitsConsumed = 8 - BIT_highbit32(lastByte); }
|
||||
bitD->bitsConsumed += (U32)(sizeof(size_t) - srcSize)*8;
|
||||
bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
|
||||
}
|
||||
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BIT_getUpperBits(size_t bitD, U32 const start)
|
||||
MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
|
||||
{
|
||||
return bitD >> start;
|
||||
return bitContainer >> start;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BIT_getMiddleBits(size_t bitD, U32 const nbBits, U32 const start)
|
||||
MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
|
||||
{
|
||||
#if defined(__BMI__) && defined(__GNUC__) /* experimental */
|
||||
return __builtin_ia32_bextr_u64(bitD, (nbBits<<8) | start );
|
||||
# if defined(__x86_64__)
|
||||
if (sizeof(bitContainer)==8)
|
||||
return _bextr_u64(bitContainer, start, nbBits);
|
||||
else
|
||||
# endif
|
||||
return _bextr_u32(bitContainer, start, nbBits);
|
||||
#else
|
||||
return (bitD >> start) & BIT_mask[nbBits];
|
||||
return (bitContainer >> start) & BIT_mask[nbBits];
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BIT_getLowerBits(size_t bitD, U32 const nbBits)
|
||||
MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
|
||||
{
|
||||
return bitD & BIT_mask[nbBits];
|
||||
return bitContainer & BIT_mask[nbBits];
|
||||
}
|
||||
|
||||
/*! BIT_lookBits() :
|
||||
* Provides next n bits from local register.
|
||||
* local register is not modified (bits are still present for next read/look).
|
||||
* local register is not modified.
|
||||
* On 32-bits, maxNbBits==24.
|
||||
* On 64-bits, maxNbBits==56.
|
||||
* @return : value extracted
|
||||
*/
|
||||
MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
#if defined(__BMI__) && defined(__GNUC__) /* experimental */
|
||||
return __builtin_ia32_bextr_u64(bitD->bitContainer, (nbBits<<8) | (64 - bitD->bitsConsumed - nbBits) );
|
||||
#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
|
||||
return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
|
||||
#else
|
||||
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
|
||||
@ -367,7 +372,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
||||
* Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ).
|
||||
* This function is safe, it guarantees it will not read beyond src buffer.
|
||||
* @return : status of `BIT_DStream_t` internal register.
|
||||
if status == unfinished, internal register is filled with >= (sizeof(size_t)*8 - 7) bits */
|
||||
if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
||||
{
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
||||
|
||||
158
contrib/libzstd/include/zstd/entropy_common.c
Normal file
158
contrib/libzstd/include/zstd/entropy_common.c
Normal file
@ -0,0 +1,158 @@
|
||||
/*
|
||||
Common functions of New Generation Entropy library
|
||||
Copyright (C) 2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*************************************************************************** */
|
||||
|
||||
/* *************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stdlib.h>
|
||||
#include "mem.h"
|
||||
#include "fse_static.h" /* FSE_MIN_TABLELOG */
|
||||
#include "error_private.h"
|
||||
#include "fse.h" /* declaration of FSE_isError, FSE_getErrorName */
|
||||
#include "huf.h" /* declaration of HUF_isError, HUF_getErrorName */
|
||||
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* FSE Error Management
|
||||
******************************************/
|
||||
unsigned FSE_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* HUF Error Management
|
||||
****************************************************************/
|
||||
unsigned HUF_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* FSE NCount encoding-decoding
|
||||
****************************************************************/
|
||||
static short FSE_abs(short a) { return a<0 ? -a : a; }
|
||||
|
||||
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*) headerBuffer;
|
||||
const BYTE* const iend = istart + hbSize;
|
||||
const BYTE* ip = istart;
|
||||
int nbBits;
|
||||
int remaining;
|
||||
int threshold;
|
||||
U32 bitStream;
|
||||
int bitCount;
|
||||
unsigned charnum = 0;
|
||||
int previous0 = 0;
|
||||
|
||||
if (hbSize < 4) return ERROR(srcSize_wrong);
|
||||
bitStream = MEM_readLE32(ip);
|
||||
nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
|
||||
if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
|
||||
bitStream >>= 4;
|
||||
bitCount = 4;
|
||||
*tableLogPtr = nbBits;
|
||||
remaining = (1<<nbBits)+1;
|
||||
threshold = 1<<nbBits;
|
||||
nbBits++;
|
||||
|
||||
while ((remaining>1) && (charnum<=*maxSVPtr)) {
|
||||
if (previous0) {
|
||||
unsigned n0 = charnum;
|
||||
while ((bitStream & 0xFFFF) == 0xFFFF) {
|
||||
n0+=24;
|
||||
if (ip < iend-5) {
|
||||
ip+=2;
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
} else {
|
||||
bitStream >>= 16;
|
||||
bitCount+=16;
|
||||
} }
|
||||
while ((bitStream & 3) == 3) {
|
||||
n0+=3;
|
||||
bitStream>>=2;
|
||||
bitCount+=2;
|
||||
}
|
||||
n0 += bitStream & 3;
|
||||
bitCount += 2;
|
||||
if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
|
||||
while (charnum < n0) normalizedCounter[charnum++] = 0;
|
||||
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
||||
ip += bitCount>>3;
|
||||
bitCount &= 7;
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
}
|
||||
else
|
||||
bitStream >>= 2;
|
||||
}
|
||||
{ short const max = (short)((2*threshold-1)-remaining);
|
||||
short count;
|
||||
|
||||
if ((bitStream & (threshold-1)) < (U32)max) {
|
||||
count = (short)(bitStream & (threshold-1));
|
||||
bitCount += nbBits-1;
|
||||
} else {
|
||||
count = (short)(bitStream & (2*threshold-1));
|
||||
if (count >= threshold) count -= max;
|
||||
bitCount += nbBits;
|
||||
}
|
||||
|
||||
count--; /* extra accuracy */
|
||||
remaining -= FSE_abs(count);
|
||||
normalizedCounter[charnum++] = count;
|
||||
previous0 = !count;
|
||||
while (remaining < threshold) {
|
||||
nbBits--;
|
||||
threshold >>= 1;
|
||||
}
|
||||
|
||||
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
||||
ip += bitCount>>3;
|
||||
bitCount &= 7;
|
||||
} else {
|
||||
bitCount -= (int)(8 * (iend - 4 - ip));
|
||||
ip = iend - 4;
|
||||
}
|
||||
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
|
||||
} } /* while ((remaining>1) && (charnum<=*maxSVPtr)) */
|
||||
if (remaining != 1) return ERROR(GENERIC);
|
||||
*maxSVPtr = charnum-1;
|
||||
|
||||
ip += (bitCount+7)>>3;
|
||||
if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong);
|
||||
return ip-istart;
|
||||
}
|
||||
@ -78,17 +78,17 @@ typedef ZSTD_ErrorCode ERR_enum;
|
||||
|
||||
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
|
||||
|
||||
ERR_STATIC ERR_enum ERR_getError(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
|
||||
ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Error Strings
|
||||
******************************************/
|
||||
|
||||
ERR_STATIC const char* ERR_getErrorName(size_t code)
|
||||
ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
|
||||
{
|
||||
static const char* notErrorCode = "Unspecified error code";
|
||||
switch( ERR_getError(code) )
|
||||
switch( code )
|
||||
{
|
||||
case PREFIX(no_error): return "No error detected";
|
||||
case PREFIX(GENERIC): return "Error (generic)";
|
||||
@ -107,10 +107,14 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
|
||||
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
|
||||
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
|
||||
case PREFIX(maxCode):
|
||||
default: return notErrorCode; /* impossible, due to ERR_getError() */
|
||||
default: return notErrorCode;
|
||||
}
|
||||
}
|
||||
|
||||
ERR_STATIC const char* ERR_getErrorName(size_t code)
|
||||
{
|
||||
return ERR_getErrorString(ERR_getErrorCode(code));
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/* ******************************************************************
|
||||
FSE : Finite State Entropy coder
|
||||
FSE : Finite State Entropy encoder
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
@ -32,35 +32,6 @@
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
#ifndef FSE_COMMONDEFS_ONLY
|
||||
|
||||
/* **************************************************************
|
||||
* Tuning parameters
|
||||
****************************************************************/
|
||||
/*!MEMORY_USAGE :
|
||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
||||
* Increasing memory usage improves compression ratio
|
||||
* Reduced memory usage can improve speed, due to cache effect
|
||||
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
|
||||
#define FSE_MAX_MEMORY_USAGE 14
|
||||
#define FSE_DEFAULT_MEMORY_USAGE 13
|
||||
|
||||
/*!FSE_MAX_SYMBOL_VALUE :
|
||||
* Maximum symbol value authorized.
|
||||
* Required for proper stack allocation */
|
||||
#define FSE_MAX_SYMBOL_VALUE 255
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* template functions type & suffix
|
||||
****************************************************************/
|
||||
#define FSE_FUNCTION_TYPE BYTE
|
||||
#define FSE_FUNCTION_EXTENSION
|
||||
#define FSE_DECODE_TYPE FSE_decode_t
|
||||
|
||||
|
||||
#endif /* !FSE_COMMONDEFS_ONLY */
|
||||
|
||||
/* **************************************************************
|
||||
* Compiler specifics
|
||||
****************************************************************/
|
||||
@ -89,21 +60,6 @@
|
||||
#include "fse_static.h"
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
* Constants
|
||||
*****************************************************************/
|
||||
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
|
||||
#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
|
||||
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
|
||||
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
|
||||
#define FSE_MIN_TABLELOG 5
|
||||
|
||||
#define FSE_TABLELOG_ABSOLUTE_MAX 15
|
||||
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
|
||||
#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Error Management
|
||||
****************************************************************/
|
||||
@ -114,7 +70,6 @@
|
||||
* Complex types
|
||||
****************************************************************/
|
||||
typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
|
||||
typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
@ -141,8 +96,6 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
|
||||
|
||||
|
||||
/* Function templates */
|
||||
static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; }
|
||||
|
||||
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
U32 const tableSize = 1 << tableLog;
|
||||
@ -151,12 +104,15 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
|
||||
U16* const tableU16 = ( (U16*) ptr) + 2;
|
||||
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
|
||||
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
|
||||
U32 const step = FSE_tableStep(tableSize);
|
||||
U32 const step = FSE_TABLESTEP(tableSize);
|
||||
U32 cumul[FSE_MAX_SYMBOL_VALUE+2];
|
||||
|
||||
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
|
||||
U32 highThreshold = tableSize-1;
|
||||
|
||||
/* CTable header */
|
||||
|
||||
|
||||
tableU16[-2] = (U16) tableLog;
|
||||
tableU16[-1] = (U16) maxSymbolValue;
|
||||
|
||||
@ -186,6 +142,7 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */
|
||||
} }
|
||||
|
||||
if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */
|
||||
}
|
||||
|
||||
@ -202,6 +159,7 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
|
||||
switch (normalizedCounter[s])
|
||||
{
|
||||
case 0: break;
|
||||
|
||||
case -1:
|
||||
case 1:
|
||||
symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog);
|
||||
@ -221,81 +179,8 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
|
||||
}
|
||||
|
||||
|
||||
FSE_DTable* FSE_createDTable (unsigned tableLog)
|
||||
{
|
||||
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
|
||||
return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
|
||||
}
|
||||
|
||||
void FSE_freeDTable (FSE_DTable* dt)
|
||||
{
|
||||
free(dt);
|
||||
}
|
||||
|
||||
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
FSE_DTableHeader DTableH;
|
||||
void* const tdPtr = dt+1; /* because dt is unsigned, 32-bits aligned on 32-bits */
|
||||
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
|
||||
const U32 tableSize = 1 << tableLog;
|
||||
const U32 tableMask = tableSize-1;
|
||||
const U32 step = FSE_tableStep(tableSize);
|
||||
U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
|
||||
U32 highThreshold = tableSize-1;
|
||||
S16 const largeLimit= (S16)(1 << (tableLog-1));
|
||||
U32 noLarge = 1;
|
||||
U32 s;
|
||||
|
||||
/* Sanity Checks */
|
||||
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
|
||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
|
||||
/* Init, lay down lowprob symbols */
|
||||
DTableH.tableLog = (U16)tableLog;
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
if (normalizedCounter[s]==-1) {
|
||||
tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
|
||||
symbolNext[s] = 1;
|
||||
} else {
|
||||
if (normalizedCounter[s] >= largeLimit) noLarge=0;
|
||||
symbolNext[s] = normalizedCounter[s];
|
||||
} }
|
||||
|
||||
/* Spread symbols */
|
||||
{ U32 position = 0;
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
int i;
|
||||
for (i=0; i<normalizedCounter[s]; i++) {
|
||||
tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
|
||||
} }
|
||||
if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
|
||||
}
|
||||
|
||||
/* Build Decoding table */
|
||||
{ U32 u;
|
||||
for (u=0; u<tableSize; u++) {
|
||||
FSE_FUNCTION_TYPE symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
|
||||
U16 nextState = symbolNext[symbol]++;
|
||||
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
|
||||
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
|
||||
} }
|
||||
|
||||
DTableH.fastMode = (U16)noLarge;
|
||||
memcpy(dt, &DTableH, sizeof(DTableH));
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
#ifndef FSE_COMMONDEFS_ONLY
|
||||
/*-****************************************
|
||||
* FSE helper functions
|
||||
******************************************/
|
||||
unsigned FSE_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* FSE NCount encoding-decoding
|
||||
@ -408,98 +293,6 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*) headerBuffer;
|
||||
const BYTE* const iend = istart + hbSize;
|
||||
const BYTE* ip = istart;
|
||||
int nbBits;
|
||||
int remaining;
|
||||
int threshold;
|
||||
U32 bitStream;
|
||||
int bitCount;
|
||||
unsigned charnum = 0;
|
||||
int previous0 = 0;
|
||||
|
||||
if (hbSize < 4) return ERROR(srcSize_wrong);
|
||||
bitStream = MEM_readLE32(ip);
|
||||
nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
|
||||
if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
|
||||
bitStream >>= 4;
|
||||
bitCount = 4;
|
||||
*tableLogPtr = nbBits;
|
||||
remaining = (1<<nbBits)+1;
|
||||
threshold = 1<<nbBits;
|
||||
nbBits++;
|
||||
|
||||
while ((remaining>1) && (charnum<=*maxSVPtr)) {
|
||||
if (previous0) {
|
||||
unsigned n0 = charnum;
|
||||
while ((bitStream & 0xFFFF) == 0xFFFF) {
|
||||
n0+=24;
|
||||
if (ip < iend-5) {
|
||||
ip+=2;
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
} else {
|
||||
bitStream >>= 16;
|
||||
bitCount+=16;
|
||||
} }
|
||||
while ((bitStream & 3) == 3) {
|
||||
n0+=3;
|
||||
bitStream>>=2;
|
||||
bitCount+=2;
|
||||
}
|
||||
n0 += bitStream & 3;
|
||||
bitCount += 2;
|
||||
if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
|
||||
while (charnum < n0) normalizedCounter[charnum++] = 0;
|
||||
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
||||
ip += bitCount>>3;
|
||||
bitCount &= 7;
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
}
|
||||
else
|
||||
bitStream >>= 2;
|
||||
}
|
||||
{ short const max = (short)((2*threshold-1)-remaining);
|
||||
short count;
|
||||
|
||||
if ((bitStream & (threshold-1)) < (U32)max) {
|
||||
count = (short)(bitStream & (threshold-1));
|
||||
bitCount += nbBits-1;
|
||||
} else {
|
||||
count = (short)(bitStream & (2*threshold-1));
|
||||
if (count >= threshold) count -= max;
|
||||
bitCount += nbBits;
|
||||
}
|
||||
|
||||
count--; /* extra accuracy */
|
||||
remaining -= FSE_abs(count);
|
||||
normalizedCounter[charnum++] = count;
|
||||
previous0 = !count;
|
||||
while (remaining < threshold) {
|
||||
nbBits--;
|
||||
threshold >>= 1;
|
||||
}
|
||||
|
||||
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
||||
ip += bitCount>>3;
|
||||
bitCount &= 7;
|
||||
} else {
|
||||
bitCount -= (int)(8 * (iend - 4 - ip));
|
||||
ip = iend - 4;
|
||||
}
|
||||
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
|
||||
} }
|
||||
if (remaining != 1) return ERROR(GENERIC);
|
||||
*maxSVPtr = charnum-1;
|
||||
|
||||
ip += (bitCount+7)>>3;
|
||||
if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong);
|
||||
return ip-istart;
|
||||
}
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Counting histogram
|
||||
@ -519,6 +312,7 @@ static size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
unsigned maxSymbolValue = *maxSymbolValuePtr;
|
||||
unsigned max=0;
|
||||
|
||||
|
||||
memset(count, 0, (maxSymbolValue+1)*sizeof(*count));
|
||||
if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; }
|
||||
|
||||
@ -542,6 +336,7 @@ static size_t FSE_count_parallel(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
unsigned maxSymbolValue = *maxSymbolValuePtr;
|
||||
unsigned max=0;
|
||||
|
||||
|
||||
U32 Counting1[256] = { 0 };
|
||||
U32 Counting2[256] = { 0 };
|
||||
U32 Counting3[256] = { 0 };
|
||||
@ -619,6 +414,7 @@ size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* FSE Compression Code
|
||||
****************************************************************/
|
||||
@ -763,6 +559,7 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
|
||||
if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
|
||||
|
||||
{ U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
|
||||
|
||||
U64 const scale = 62 - tableLog;
|
||||
U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */
|
||||
U64 const vStep = 1ULL<<(scale-20);
|
||||
@ -839,11 +636,13 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
|
||||
|
||||
/* Build Symbol Transformation Table */
|
||||
{ const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
|
||||
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
symbolTT[s].deltaNbBits = deltaNbBits;
|
||||
symbolTT[s].deltaFindState = s-1;
|
||||
} }
|
||||
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -878,6 +677,8 @@ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
|
||||
const BYTE* const istart = (const BYTE*) src;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* ip=iend;
|
||||
|
||||
|
||||
BIT_CStream_t bitC;
|
||||
FSE_CState_t CState1, CState2;
|
||||
|
||||
@ -908,6 +709,7 @@ static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
|
||||
|
||||
/* 2 or 4 encoding per loop */
|
||||
for ( ; ip>istart ; ) {
|
||||
|
||||
FSE_encodeSymbol(&bitC, &CState2, *--ip);
|
||||
|
||||
if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */
|
||||
@ -998,162 +800,4 @@ size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize)
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Decompression (Byte symbols)
|
||||
*********************************************************/
|
||||
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
|
||||
{
|
||||
void* ptr = dt;
|
||||
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
|
||||
void* dPtr = dt + 1;
|
||||
FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
|
||||
|
||||
DTableH->tableLog = 0;
|
||||
DTableH->fastMode = 0;
|
||||
|
||||
cell->newState = 0;
|
||||
cell->symbol = symbolValue;
|
||||
cell->nbBits = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
|
||||
{
|
||||
void* ptr = dt;
|
||||
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
|
||||
void* dPtr = dt + 1;
|
||||
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
|
||||
const unsigned tableSize = 1 << nbBits;
|
||||
const unsigned tableMask = tableSize - 1;
|
||||
const unsigned maxSymbolValue = tableMask;
|
||||
unsigned s;
|
||||
|
||||
/* Sanity checks */
|
||||
if (nbBits < 1) return ERROR(GENERIC); /* min size */
|
||||
|
||||
/* Build Decoding Table */
|
||||
DTableH->tableLog = (U16)nbBits;
|
||||
DTableH->fastMode = 1;
|
||||
for (s=0; s<=maxSymbolValue; s++) {
|
||||
dinfo[s].newState = 0;
|
||||
dinfo[s].symbol = (BYTE)s;
|
||||
dinfo[s].nbBits = (BYTE)nbBits;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const FSE_DTable* dt, const unsigned fast)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const omax = op + maxDstSize;
|
||||
BYTE* const olimit = omax-3;
|
||||
|
||||
BIT_DStream_t bitD;
|
||||
FSE_DState_t state1;
|
||||
FSE_DState_t state2;
|
||||
size_t errorCode;
|
||||
|
||||
/* Init */
|
||||
errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */
|
||||
if (FSE_isError(errorCode)) return errorCode;
|
||||
|
||||
FSE_initDState(&state1, &bitD, dt);
|
||||
FSE_initDState(&state2, &bitD, dt);
|
||||
|
||||
#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
|
||||
|
||||
/* 4 symbols per loop */
|
||||
for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op<olimit) ; op+=4) {
|
||||
op[0] = FSE_GETSYMBOL(&state1);
|
||||
|
||||
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
BIT_reloadDStream(&bitD);
|
||||
|
||||
op[1] = FSE_GETSYMBOL(&state2);
|
||||
|
||||
if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
{ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
|
||||
|
||||
op[2] = FSE_GETSYMBOL(&state1);
|
||||
|
||||
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
BIT_reloadDStream(&bitD);
|
||||
|
||||
op[3] = FSE_GETSYMBOL(&state2);
|
||||
}
|
||||
|
||||
/* tail */
|
||||
/* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
|
||||
while (1) {
|
||||
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
|
||||
|
||||
*op++ = FSE_GETSYMBOL(&state1);
|
||||
|
||||
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
|
||||
*op++ = FSE_GETSYMBOL(&state2);
|
||||
break;
|
||||
}
|
||||
|
||||
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
|
||||
|
||||
*op++ = FSE_GETSYMBOL(&state2);
|
||||
|
||||
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
|
||||
*op++ = FSE_GETSYMBOL(&state1);
|
||||
break;
|
||||
} }
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const FSE_DTable* dt)
|
||||
{
|
||||
const void* ptr = dt;
|
||||
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
|
||||
const U32 fastMode = DTableH->fastMode;
|
||||
|
||||
/* select fast mode (static) */
|
||||
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
|
||||
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*)cSrc;
|
||||
const BYTE* ip = istart;
|
||||
short counting[FSE_MAX_SYMBOL_VALUE+1];
|
||||
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
|
||||
unsigned tableLog;
|
||||
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
|
||||
size_t errorCode;
|
||||
|
||||
if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */
|
||||
|
||||
/* normal FSE decoding mode */
|
||||
errorCode = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
|
||||
if (FSE_isError(errorCode)) return errorCode;
|
||||
if (errorCode >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */
|
||||
ip += errorCode;
|
||||
cSrcSize -= errorCode;
|
||||
|
||||
errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog);
|
||||
if (FSE_isError(errorCode)) return errorCode;
|
||||
|
||||
/* always return, even if it is an error code */
|
||||
return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt);
|
||||
}
|
||||
|
||||
|
||||
|
||||
#endif /* FSE_COMMONDEFS_ONLY */
|
||||
330
contrib/libzstd/include/zstd/fse_decompress.c
Normal file
330
contrib/libzstd/include/zstd/fse_decompress.c
Normal file
@ -0,0 +1,330 @@
|
||||
/* ******************************************************************
|
||||
FSE : Finite State Entropy decoder
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Compiler specifics
|
||||
****************************************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define FORCE_INLINE static __forceinline
|
||||
# include <intrin.h> /* For Visual 2005 */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
|
||||
#else
|
||||
# ifdef __GNUC__
|
||||
# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
# define FORCE_INLINE static inline __attribute__((always_inline))
|
||||
# else
|
||||
# define FORCE_INLINE static inline
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Includes
|
||||
****************************************************************/
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include <stdio.h> /* printf (debug) */
|
||||
#include "bitstream.h"
|
||||
#include "fse_static.h"
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Error Management
|
||||
****************************************************************/
|
||||
#define FSE_isError ERR_isError
|
||||
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Complex types
|
||||
****************************************************************/
|
||||
typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Templates
|
||||
****************************************************************/
|
||||
/*
|
||||
designed to be included
|
||||
for type-specific functions (template emulation in C)
|
||||
Objective is to write these functions only once, for improved maintenance
|
||||
*/
|
||||
|
||||
/* safety checks */
|
||||
#ifndef FSE_FUNCTION_EXTENSION
|
||||
# error "FSE_FUNCTION_EXTENSION must be defined"
|
||||
#endif
|
||||
#ifndef FSE_FUNCTION_TYPE
|
||||
# error "FSE_FUNCTION_TYPE must be defined"
|
||||
#endif
|
||||
|
||||
/* Function names */
|
||||
#define FSE_CAT(X,Y) X##Y
|
||||
#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
|
||||
#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
|
||||
|
||||
|
||||
/* Function templates */
|
||||
FSE_DTable* FSE_createDTable (unsigned tableLog)
|
||||
{
|
||||
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
|
||||
return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
|
||||
}
|
||||
|
||||
void FSE_freeDTable (FSE_DTable* dt)
|
||||
{
|
||||
free(dt);
|
||||
}
|
||||
|
||||
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
|
||||
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
|
||||
U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
|
||||
|
||||
U32 const maxSV1 = maxSymbolValue + 1;
|
||||
U32 const tableSize = 1 << tableLog;
|
||||
U32 highThreshold = tableSize-1;
|
||||
|
||||
/* Sanity Checks */
|
||||
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
|
||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
|
||||
/* Init, lay down lowprob symbols */
|
||||
{ FSE_DTableHeader DTableH;
|
||||
DTableH.tableLog = (U16)tableLog;
|
||||
DTableH.fastMode = 1;
|
||||
{ S16 const largeLimit= (S16)(1 << (tableLog-1));
|
||||
U32 s;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
if (normalizedCounter[s]==-1) {
|
||||
tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
|
||||
symbolNext[s] = 1;
|
||||
} else {
|
||||
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
|
||||
symbolNext[s] = normalizedCounter[s];
|
||||
} } }
|
||||
memcpy(dt, &DTableH, sizeof(DTableH));
|
||||
}
|
||||
|
||||
/* Spread symbols */
|
||||
{ U32 const tableMask = tableSize-1;
|
||||
U32 const step = FSE_TABLESTEP(tableSize);
|
||||
U32 s, position = 0;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
int i;
|
||||
for (i=0; i<normalizedCounter[s]; i++) {
|
||||
tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
|
||||
} }
|
||||
|
||||
if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
|
||||
}
|
||||
|
||||
/* Build Decoding table */
|
||||
{ U32 u;
|
||||
for (u=0; u<tableSize; u++) {
|
||||
FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
|
||||
U16 nextState = symbolNext[symbol]++;
|
||||
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
|
||||
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
|
||||
} }
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#ifndef FSE_COMMONDEFS_ONLY
|
||||
|
||||
/*-*******************************************************
|
||||
* Decompression (Byte symbols)
|
||||
*********************************************************/
|
||||
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
|
||||
{
|
||||
void* ptr = dt;
|
||||
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
|
||||
void* dPtr = dt + 1;
|
||||
FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
|
||||
|
||||
DTableH->tableLog = 0;
|
||||
DTableH->fastMode = 0;
|
||||
|
||||
cell->newState = 0;
|
||||
cell->symbol = symbolValue;
|
||||
cell->nbBits = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
|
||||
{
|
||||
void* ptr = dt;
|
||||
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
|
||||
void* dPtr = dt + 1;
|
||||
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
|
||||
const unsigned tableSize = 1 << nbBits;
|
||||
const unsigned tableMask = tableSize - 1;
|
||||
const unsigned maxSV1 = tableMask+1;
|
||||
unsigned s;
|
||||
|
||||
/* Sanity checks */
|
||||
if (nbBits < 1) return ERROR(GENERIC); /* min size */
|
||||
|
||||
/* Build Decoding Table */
|
||||
DTableH->tableLog = (U16)nbBits;
|
||||
DTableH->fastMode = 1;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
dinfo[s].newState = 0;
|
||||
dinfo[s].symbol = (BYTE)s;
|
||||
dinfo[s].nbBits = (BYTE)nbBits;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const FSE_DTable* dt, const unsigned fast)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const omax = op + maxDstSize;
|
||||
BYTE* const olimit = omax-3;
|
||||
|
||||
BIT_DStream_t bitD;
|
||||
FSE_DState_t state1;
|
||||
FSE_DState_t state2;
|
||||
|
||||
/* Init */
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */
|
||||
if (FSE_isError(errorCode)) return errorCode; }
|
||||
|
||||
FSE_initDState(&state1, &bitD, dt);
|
||||
FSE_initDState(&state2, &bitD, dt);
|
||||
|
||||
#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
|
||||
|
||||
/* 4 symbols per loop */
|
||||
for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op<olimit) ; op+=4) {
|
||||
op[0] = FSE_GETSYMBOL(&state1);
|
||||
|
||||
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
BIT_reloadDStream(&bitD);
|
||||
|
||||
op[1] = FSE_GETSYMBOL(&state2);
|
||||
|
||||
if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
{ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
|
||||
|
||||
op[2] = FSE_GETSYMBOL(&state1);
|
||||
|
||||
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
BIT_reloadDStream(&bitD);
|
||||
|
||||
op[3] = FSE_GETSYMBOL(&state2);
|
||||
}
|
||||
|
||||
/* tail */
|
||||
/* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
|
||||
while (1) {
|
||||
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
|
||||
|
||||
*op++ = FSE_GETSYMBOL(&state1);
|
||||
|
||||
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
|
||||
*op++ = FSE_GETSYMBOL(&state2);
|
||||
break;
|
||||
}
|
||||
|
||||
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
|
||||
|
||||
*op++ = FSE_GETSYMBOL(&state2);
|
||||
|
||||
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
|
||||
*op++ = FSE_GETSYMBOL(&state1);
|
||||
break;
|
||||
} }
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const FSE_DTable* dt)
|
||||
{
|
||||
const void* ptr = dt;
|
||||
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
|
||||
const U32 fastMode = DTableH->fastMode;
|
||||
|
||||
/* select fast mode (static) */
|
||||
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
|
||||
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*)cSrc;
|
||||
const BYTE* ip = istart;
|
||||
short counting[FSE_MAX_SYMBOL_VALUE+1];
|
||||
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
|
||||
unsigned tableLog;
|
||||
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
|
||||
|
||||
if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */
|
||||
|
||||
/* normal FSE decoding mode */
|
||||
{ size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
|
||||
if (FSE_isError(NCountLength)) return NCountLength;
|
||||
if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */
|
||||
ip += NCountLength;
|
||||
cSrcSize -= NCountLength;
|
||||
}
|
||||
|
||||
{ size_t const errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog);
|
||||
if (FSE_isError(errorCode)) return errorCode; }
|
||||
|
||||
return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */
|
||||
}
|
||||
|
||||
|
||||
|
||||
#endif /* FSE_COMMONDEFS_ONLY */
|
||||
@ -334,6 +334,54 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
||||
}
|
||||
|
||||
|
||||
|
||||
#ifndef FSE_COMMONDEFS_ONLY
|
||||
|
||||
/* **************************************************************
|
||||
* Tuning parameters
|
||||
****************************************************************/
|
||||
/*!MEMORY_USAGE :
|
||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
||||
* Increasing memory usage improves compression ratio
|
||||
* Reduced memory usage can improve speed, due to cache effect
|
||||
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
|
||||
#define FSE_MAX_MEMORY_USAGE 14
|
||||
#define FSE_DEFAULT_MEMORY_USAGE 13
|
||||
|
||||
/*!FSE_MAX_SYMBOL_VALUE :
|
||||
* Maximum symbol value authorized.
|
||||
* Required for proper stack allocation */
|
||||
#define FSE_MAX_SYMBOL_VALUE 255
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* template functions type & suffix
|
||||
****************************************************************/
|
||||
#define FSE_FUNCTION_TYPE BYTE
|
||||
#define FSE_FUNCTION_EXTENSION
|
||||
#define FSE_DECODE_TYPE FSE_decode_t
|
||||
|
||||
|
||||
#endif /* !FSE_COMMONDEFS_ONLY */
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
* Constants
|
||||
*****************************************************************/
|
||||
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
|
||||
#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
|
||||
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
|
||||
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
|
||||
#define FSE_MIN_TABLELOG 5
|
||||
|
||||
#define FSE_TABLELOG_ABSOLUTE_MAX 15
|
||||
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
|
||||
#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
|
||||
#endif
|
||||
|
||||
#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/* ******************************************************************
|
||||
Huff0 : Huffman coder, part of New Generation Entropy library
|
||||
Huffman coder, part of New Generation Entropy library
|
||||
header file
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
@ -31,8 +31,8 @@
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef HUFF0_H
|
||||
#define HUFF0_H
|
||||
#ifndef HUF_H
|
||||
#define HUF_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
@ -46,7 +46,7 @@ extern "C" {
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Huff0 simple functions
|
||||
* HUF simple functions
|
||||
******************************************/
|
||||
size_t HUF_compress(void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
@ -63,7 +63,7 @@ HUF_compress() :
|
||||
if HUF_isError(return), compression failed (more details using HUF_getErrorName())
|
||||
|
||||
HUF_decompress() :
|
||||
Decompress Huff0 data from buffer 'cSrc', of size 'cSrcSize',
|
||||
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated destination buffer 'dst', of size 'dstSize'.
|
||||
`dstSize` : must be the **exact** size of original (uncompressed) data.
|
||||
Note : in contrast with FSE, HUF_decompress can regenerate
|
||||
@ -94,4 +94,4 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* HUFF0_H */
|
||||
#endif /* HUF_H */
|
||||
560
contrib/libzstd/include/zstd/huf_compress.c
Normal file
560
contrib/libzstd/include/zstd/huf_compress.c
Normal file
@ -0,0 +1,560 @@
|
||||
/* ******************************************************************
|
||||
Huffman encoder, part of New Generation Entropy library
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
/* **************************************************************
|
||||
* Compiler specifics
|
||||
****************************************************************/
|
||||
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
/* inline is defined */
|
||||
#elif defined(_MSC_VER)
|
||||
# define inline __inline
|
||||
#else
|
||||
# define inline /* disable inline */
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define FORCE_INLINE static __forceinline
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#else
|
||||
# ifdef __GNUC__
|
||||
# define FORCE_INLINE static inline __attribute__((always_inline))
|
||||
# else
|
||||
# define FORCE_INLINE static inline
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Includes
|
||||
****************************************************************/
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include <stdio.h> /* printf (debug) */
|
||||
#include "huf_static.h"
|
||||
#include "bitstream.h"
|
||||
#include "fse.h" /* header compression */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Error Management
|
||||
****************************************************************/
|
||||
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
|
||||
|
||||
/* *******************************************************
|
||||
* HUF : Huffman block compression
|
||||
*********************************************************/
|
||||
struct HUF_CElt_s {
|
||||
U16 val;
|
||||
BYTE nbBits;
|
||||
}; /* typedef'd to HUF_CElt within huf_static.h */
|
||||
|
||||
typedef struct nodeElt_s {
|
||||
U32 count;
|
||||
U16 parent;
|
||||
BYTE byte;
|
||||
BYTE nbBits;
|
||||
} nodeElt;
|
||||
|
||||
/*! HUF_writeCTable() :
|
||||
`CTable` : huffman tree to save, using huf representation.
|
||||
@return : size of saved CTable */
|
||||
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
||||
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
|
||||
{
|
||||
BYTE bitsToWeight[HUF_MAX_TABLELOG + 1];
|
||||
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
|
||||
U32 n;
|
||||
BYTE* op = (BYTE*)dst;
|
||||
size_t size;
|
||||
|
||||
/* check conditions */
|
||||
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1)
|
||||
return ERROR(GENERIC);
|
||||
|
||||
/* convert to weight */
|
||||
bitsToWeight[0] = 0;
|
||||
for (n=1; n<=huffLog; n++)
|
||||
bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
|
||||
for (n=0; n<maxSymbolValue; n++)
|
||||
huffWeight[n] = bitsToWeight[CTable[n].nbBits];
|
||||
|
||||
size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); /* don't need last symbol stat : implied */
|
||||
if (HUF_isError(size)) return size;
|
||||
if (size >= 128) return ERROR(GENERIC); /* should never happen, since maxSymbolValue <= 255 */
|
||||
if ((size <= 1) || (size >= maxSymbolValue/2)) {
|
||||
if (size==1) { /* RLE */
|
||||
/* only possible case : serie of 1 (because there are at least 2) */
|
||||
/* can only be 2^n or (2^n-1), otherwise not an huffman tree */
|
||||
BYTE code;
|
||||
switch(maxSymbolValue)
|
||||
{
|
||||
case 1: code = 0; break;
|
||||
case 2: code = 1; break;
|
||||
case 3: code = 2; break;
|
||||
case 4: code = 3; break;
|
||||
case 7: code = 4; break;
|
||||
case 8: code = 5; break;
|
||||
case 15: code = 6; break;
|
||||
case 16: code = 7; break;
|
||||
case 31: code = 8; break;
|
||||
case 32: code = 9; break;
|
||||
case 63: code = 10; break;
|
||||
case 64: code = 11; break;
|
||||
case 127: code = 12; break;
|
||||
case 128: code = 13; break;
|
||||
default : return ERROR(corruption_detected);
|
||||
}
|
||||
op[0] = (BYTE)(255-13 + code);
|
||||
return 1;
|
||||
}
|
||||
/* Not compressible */
|
||||
if (maxSymbolValue > (241-128)) return ERROR(GENERIC); /* not implemented (not possible with current format) */
|
||||
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
|
||||
op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1));
|
||||
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
|
||||
for (n=0; n<maxSymbolValue; n+=2)
|
||||
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
|
||||
return ((maxSymbolValue+1)/2) + 1;
|
||||
}
|
||||
|
||||
/* normal header case */
|
||||
op[0] = (BYTE)size;
|
||||
return size+1;
|
||||
}
|
||||
|
||||
|
||||
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
|
||||
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
|
||||
U32 tableLog = 0;
|
||||
size_t readSize;
|
||||
U32 nbSymbols = 0;
|
||||
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
/* get symbol weights */
|
||||
readSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
||||
if (HUF_isError(readSize)) return readSize;
|
||||
|
||||
/* check result */
|
||||
if (tableLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
|
||||
|
||||
/* Prepare base value per rank */
|
||||
{ U32 n, nextRankStart = 0;
|
||||
for (n=1; n<=tableLog; n++) {
|
||||
U32 current = nextRankStart;
|
||||
nextRankStart += (rankVal[n] << (n-1));
|
||||
rankVal[n] = current;
|
||||
} }
|
||||
|
||||
/* fill nbBits */
|
||||
{ U32 n; for (n=0; n<nbSymbols; n++) {
|
||||
const U32 w = huffWeight[n];
|
||||
CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
|
||||
}}
|
||||
|
||||
/* fill val */
|
||||
{ U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0};
|
||||
U16 valPerRank[HUF_MAX_TABLELOG+1] = {0};
|
||||
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
|
||||
/* determine stating value per rank */
|
||||
{ U16 min = 0;
|
||||
U32 n; for (n=HUF_MAX_TABLELOG; n>0; n--) {
|
||||
valPerRank[n] = min; /* get starting value within each rank */
|
||||
min += nbPerRank[n];
|
||||
min >>= 1;
|
||||
} }
|
||||
/* assign value within rank, symbol order */
|
||||
{ U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; }
|
||||
}
|
||||
|
||||
return readSize;
|
||||
}
|
||||
|
||||
|
||||
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
||||
{
|
||||
const U32 largestBits = huffNode[lastNonNull].nbBits;
|
||||
if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */
|
||||
|
||||
/* there are several too large elements (at least >= 2) */
|
||||
{ int totalCost = 0;
|
||||
const U32 baseCost = 1 << (largestBits - maxNbBits);
|
||||
U32 n = lastNonNull;
|
||||
|
||||
while (huffNode[n].nbBits > maxNbBits) {
|
||||
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
|
||||
huffNode[n].nbBits = (BYTE)maxNbBits;
|
||||
n --;
|
||||
} /* n stops at huffNode[n].nbBits <= maxNbBits */
|
||||
while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */
|
||||
|
||||
/* renorm totalCost */
|
||||
totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */
|
||||
|
||||
/* repay normalized cost */
|
||||
{ U32 const noSymbol = 0xF0F0F0F0;
|
||||
U32 rankLast[HUF_MAX_TABLELOG+1];
|
||||
int pos;
|
||||
|
||||
/* Get pos of last (smallest) symbol per rank */
|
||||
memset(rankLast, 0xF0, sizeof(rankLast));
|
||||
{ U32 currentNbBits = maxNbBits;
|
||||
for (pos=n ; pos >= 0; pos--) {
|
||||
if (huffNode[pos].nbBits >= currentNbBits) continue;
|
||||
currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */
|
||||
rankLast[maxNbBits-currentNbBits] = pos;
|
||||
} }
|
||||
|
||||
while (totalCost > 0) {
|
||||
U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
|
||||
for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
|
||||
U32 highPos = rankLast[nBitsToDecrease];
|
||||
U32 lowPos = rankLast[nBitsToDecrease-1];
|
||||
if (highPos == noSymbol) continue;
|
||||
if (lowPos == noSymbol) break;
|
||||
{ U32 const highTotal = huffNode[highPos].count;
|
||||
U32 const lowTotal = 2 * huffNode[lowPos].count;
|
||||
if (highTotal <= lowTotal) break;
|
||||
} }
|
||||
/* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
|
||||
while ((nBitsToDecrease<=HUF_MAX_TABLELOG) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
|
||||
nBitsToDecrease ++;
|
||||
totalCost -= 1 << (nBitsToDecrease-1);
|
||||
if (rankLast[nBitsToDecrease-1] == noSymbol)
|
||||
rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */
|
||||
huffNode[rankLast[nBitsToDecrease]].nbBits ++;
|
||||
if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
|
||||
rankLast[nBitsToDecrease] = noSymbol;
|
||||
else {
|
||||
rankLast[nBitsToDecrease]--;
|
||||
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
|
||||
rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
|
||||
} } /* while (totalCost > 0) */
|
||||
|
||||
while (totalCost < 0) { /* Sometimes, cost correction overshoot */
|
||||
if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */
|
||||
while (huffNode[n].nbBits == maxNbBits) n--;
|
||||
huffNode[n+1].nbBits--;
|
||||
rankLast[1] = n+1;
|
||||
totalCost++;
|
||||
continue;
|
||||
}
|
||||
huffNode[ rankLast[1] + 1 ].nbBits--;
|
||||
rankLast[1]++;
|
||||
totalCost ++;
|
||||
} } } /* there are several too large elements (at least >= 2) */
|
||||
|
||||
return maxNbBits;
|
||||
}
|
||||
|
||||
|
||||
typedef struct {
|
||||
U32 base;
|
||||
U32 current;
|
||||
} rankPos;
|
||||
|
||||
static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
|
||||
{
|
||||
rankPos rank[32];
|
||||
U32 n;
|
||||
|
||||
memset(rank, 0, sizeof(rank));
|
||||
for (n=0; n<=maxSymbolValue; n++) {
|
||||
U32 r = BIT_highbit32(count[n] + 1);
|
||||
rank[r].base ++;
|
||||
}
|
||||
for (n=30; n>0; n--) rank[n-1].base += rank[n].base;
|
||||
for (n=0; n<32; n++) rank[n].current = rank[n].base;
|
||||
for (n=0; n<=maxSymbolValue; n++) {
|
||||
U32 const c = count[n];
|
||||
U32 const r = BIT_highbit32(c+1) + 1;
|
||||
U32 pos = rank[r].current++;
|
||||
while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--;
|
||||
huffNode[pos].count = c;
|
||||
huffNode[pos].byte = (BYTE)n;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#define STARTNODE (HUF_MAX_SYMBOL_VALUE+1)
|
||||
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
|
||||
{
|
||||
nodeElt huffNode0[2*HUF_MAX_SYMBOL_VALUE+1 +1];
|
||||
nodeElt* huffNode = huffNode0 + 1;
|
||||
U32 n, nonNullRank;
|
||||
int lowS, lowN;
|
||||
U16 nodeNb = STARTNODE;
|
||||
U32 nodeRoot;
|
||||
|
||||
/* safety checks */
|
||||
if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG;
|
||||
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return ERROR(GENERIC);
|
||||
memset(huffNode0, 0, sizeof(huffNode0));
|
||||
|
||||
/* sort, decreasing order */
|
||||
HUF_sort(huffNode, count, maxSymbolValue);
|
||||
|
||||
/* init for parents */
|
||||
nonNullRank = maxSymbolValue;
|
||||
while(huffNode[nonNullRank].count == 0) nonNullRank--;
|
||||
lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
|
||||
huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
|
||||
huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
|
||||
nodeNb++; lowS-=2;
|
||||
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
|
||||
huffNode0[0].count = (U32)(1U<<31);
|
||||
|
||||
/* create parents */
|
||||
while (nodeNb <= nodeRoot) {
|
||||
U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
|
||||
U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
|
||||
huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
|
||||
huffNode[n1].parent = huffNode[n2].parent = nodeNb;
|
||||
nodeNb++;
|
||||
}
|
||||
|
||||
/* distribute weights (unlimited tree height) */
|
||||
huffNode[nodeRoot].nbBits = 0;
|
||||
for (n=nodeRoot-1; n>=STARTNODE; n--)
|
||||
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
|
||||
for (n=0; n<=nonNullRank; n++)
|
||||
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
|
||||
|
||||
/* enforce maxTableLog */
|
||||
maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
|
||||
|
||||
/* fill result into tree (val, nbBits) */
|
||||
{ U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0};
|
||||
U16 valPerRank[HUF_MAX_TABLELOG+1] = {0};
|
||||
if (maxNbBits > HUF_MAX_TABLELOG) return ERROR(GENERIC); /* check fit into table */
|
||||
for (n=0; n<=nonNullRank; n++)
|
||||
nbPerRank[huffNode[n].nbBits]++;
|
||||
/* determine stating value per rank */
|
||||
{ U16 min = 0;
|
||||
for (n=maxNbBits; n>0; n--) {
|
||||
valPerRank[n] = min; /* get starting value within each rank */
|
||||
min += nbPerRank[n];
|
||||
min >>= 1;
|
||||
} }
|
||||
for (n=0; n<=maxSymbolValue; n++)
|
||||
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
|
||||
for (n=0; n<=maxSymbolValue; n++)
|
||||
tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
|
||||
}
|
||||
|
||||
return maxNbBits;
|
||||
}
|
||||
|
||||
static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
|
||||
{
|
||||
BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
|
||||
}
|
||||
|
||||
size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
|
||||
|
||||
#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
|
||||
|
||||
#define HUF_FLUSHBITS_1(stream) \
|
||||
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) HUF_FLUSHBITS(stream)
|
||||
|
||||
#define HUF_FLUSHBITS_2(stream) \
|
||||
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) HUF_FLUSHBITS(stream)
|
||||
|
||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
size_t n;
|
||||
const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
|
||||
BIT_CStream_t bitC;
|
||||
|
||||
/* init */
|
||||
if (dstSize < 8) return 0; /* not enough space to compress */
|
||||
{ size_t const errorCode = BIT_initCStream(&bitC, op, oend-op);
|
||||
if (HUF_isError(errorCode)) return 0; }
|
||||
|
||||
n = srcSize & ~3; /* join to mod 4 */
|
||||
switch (srcSize & 3)
|
||||
{
|
||||
case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable);
|
||||
HUF_FLUSHBITS_2(&bitC);
|
||||
case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
|
||||
HUF_FLUSHBITS_1(&bitC);
|
||||
case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
|
||||
HUF_FLUSHBITS(&bitC);
|
||||
case 0 :
|
||||
default: ;
|
||||
}
|
||||
|
||||
for (; n>0; n-=4) { /* note : n&3==0 at this stage */
|
||||
HUF_encodeSymbol(&bitC, ip[n- 1], CTable);
|
||||
HUF_FLUSHBITS_1(&bitC);
|
||||
HUF_encodeSymbol(&bitC, ip[n- 2], CTable);
|
||||
HUF_FLUSHBITS_2(&bitC);
|
||||
HUF_encodeSymbol(&bitC, ip[n- 3], CTable);
|
||||
HUF_FLUSHBITS_1(&bitC);
|
||||
HUF_encodeSymbol(&bitC, ip[n- 4], CTable);
|
||||
HUF_FLUSHBITS(&bitC);
|
||||
}
|
||||
|
||||
return BIT_closeCStream(&bitC);
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
||||
{
|
||||
size_t segmentSize = (srcSize+3)/4; /* first 3 segments */
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
const BYTE* const iend = ip + srcSize;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
size_t errorCode;
|
||||
|
||||
if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */
|
||||
if (srcSize < 12) return 0; /* no saving possible : too small input */
|
||||
op += 6; /* jumpTable */
|
||||
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
MEM_writeLE16(ostart, (U16)errorCode);
|
||||
|
||||
ip += segmentSize;
|
||||
op += errorCode;
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
MEM_writeLE16(ostart+2, (U16)errorCode);
|
||||
|
||||
ip += segmentSize;
|
||||
op += errorCode;
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
MEM_writeLE16(ostart+4, (U16)errorCode);
|
||||
|
||||
ip += segmentSize;
|
||||
op += errorCode;
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
|
||||
op += errorCode;
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
static size_t HUF_compress_internal (
|
||||
void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
unsigned singleStream)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
|
||||
U32 count[HUF_MAX_SYMBOL_VALUE+1];
|
||||
HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1];
|
||||
size_t errorCode;
|
||||
|
||||
/* checks & inits */
|
||||
if (srcSize < 1) return 0; /* Uncompressed - note : 1 means rle, so first byte must be correct */
|
||||
if (dstSize < 1) return 0; /* not compressible within dst budget */
|
||||
if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */
|
||||
if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE;
|
||||
if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG;
|
||||
|
||||
/* Scan input and build symbol stats */
|
||||
errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }
|
||||
if (errorCode <= (srcSize >> 7)+1) return 0; /* Heuristic : not compressible enough */
|
||||
|
||||
/* Build Huffman Tree */
|
||||
errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
huffLog = (U32)errorCode;
|
||||
|
||||
/* Write table description header */
|
||||
errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode + 12 >= srcSize) return 0; /* not useful to try compression */
|
||||
op += errorCode;
|
||||
|
||||
/* Compress */
|
||||
if (singleStream)
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */
|
||||
else
|
||||
errorCode = HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
op += errorCode;
|
||||
|
||||
/* check compressibility */
|
||||
if ((size_t)(op-ostart) >= srcSize-1)
|
||||
return 0;
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_compress1X (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1);
|
||||
}
|
||||
|
||||
size_t HUF_compress2 (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_DEFAULT_TABLELOG);
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
/* ******************************************************************
|
||||
Huff0 : Huffman coder, part of New Generation Entropy library
|
||||
Huffman decoder, part of New Generation Entropy library
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
@ -28,7 +28,7 @@
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE+Huff0 source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
@ -62,524 +62,21 @@
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include <stdio.h> /* printf (debug) */
|
||||
#include "huff0_static.h"
|
||||
#include "huf_static.h"
|
||||
#include "bitstream.h"
|
||||
#include "fse.h" /* header compression */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Constants
|
||||
****************************************************************/
|
||||
#define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
|
||||
#define HUF_MAX_TABLELOG 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
|
||||
#define HUF_DEFAULT_TABLELOG HUF_MAX_TABLELOG /* tableLog by default, when not specified */
|
||||
#define HUF_MAX_SYMBOL_VALUE 255
|
||||
#if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG)
|
||||
# error "HUF_MAX_TABLELOG is too large !"
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Error Management
|
||||
****************************************************************/
|
||||
unsigned HUF_isError(size_t code) { return ERR_isError(code); }
|
||||
const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
|
||||
|
||||
/* *******************************************************
|
||||
* Huff0 : Huffman block compression
|
||||
*********************************************************/
|
||||
struct HUF_CElt_s {
|
||||
U16 val;
|
||||
BYTE nbBits;
|
||||
}; /* typedef'd to HUF_CElt within huff0_static.h */
|
||||
|
||||
typedef struct nodeElt_s {
|
||||
U32 count;
|
||||
U16 parent;
|
||||
BYTE byte;
|
||||
BYTE nbBits;
|
||||
} nodeElt;
|
||||
|
||||
/*! HUF_writeCTable() :
|
||||
`CTable` : huffman tree to save, using huff0 representation.
|
||||
@return : size of saved CTable */
|
||||
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
||||
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
|
||||
{
|
||||
BYTE bitsToWeight[HUF_MAX_TABLELOG + 1];
|
||||
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
|
||||
U32 n;
|
||||
BYTE* op = (BYTE*)dst;
|
||||
size_t size;
|
||||
|
||||
/* check conditions */
|
||||
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1)
|
||||
return ERROR(GENERIC);
|
||||
|
||||
/* convert to weight */
|
||||
bitsToWeight[0] = 0;
|
||||
for (n=1; n<=huffLog; n++)
|
||||
bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
|
||||
for (n=0; n<maxSymbolValue; n++)
|
||||
huffWeight[n] = bitsToWeight[CTable[n].nbBits];
|
||||
|
||||
size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); /* don't need last symbol stat : implied */
|
||||
if (HUF_isError(size)) return size;
|
||||
if (size >= 128) return ERROR(GENERIC); /* should never happen, since maxSymbolValue <= 255 */
|
||||
if ((size <= 1) || (size >= maxSymbolValue/2)) {
|
||||
if (size==1) { /* RLE */
|
||||
/* only possible case : serie of 1 (because there are at least 2) */
|
||||
/* can only be 2^n or (2^n-1), otherwise not an huffman tree */
|
||||
BYTE code;
|
||||
switch(maxSymbolValue)
|
||||
{
|
||||
case 1: code = 0; break;
|
||||
case 2: code = 1; break;
|
||||
case 3: code = 2; break;
|
||||
case 4: code = 3; break;
|
||||
case 7: code = 4; break;
|
||||
case 8: code = 5; break;
|
||||
case 15: code = 6; break;
|
||||
case 16: code = 7; break;
|
||||
case 31: code = 8; break;
|
||||
case 32: code = 9; break;
|
||||
case 63: code = 10; break;
|
||||
case 64: code = 11; break;
|
||||
case 127: code = 12; break;
|
||||
case 128: code = 13; break;
|
||||
default : return ERROR(corruption_detected);
|
||||
}
|
||||
op[0] = (BYTE)(255-13 + code);
|
||||
return 1;
|
||||
}
|
||||
/* Not compressible */
|
||||
if (maxSymbolValue > (241-128)) return ERROR(GENERIC); /* not implemented (not possible with current format) */
|
||||
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
|
||||
op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1));
|
||||
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
|
||||
for (n=0; n<maxSymbolValue; n+=2)
|
||||
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
|
||||
return ((maxSymbolValue+1)/2) + 1;
|
||||
}
|
||||
|
||||
/* normal header case */
|
||||
op[0] = (BYTE)size;
|
||||
return size+1;
|
||||
}
|
||||
|
||||
|
||||
static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
|
||||
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
|
||||
U32 tableLog = 0;
|
||||
size_t readSize;
|
||||
U32 nbSymbols = 0;
|
||||
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
/* get symbol weights */
|
||||
readSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
||||
if (HUF_isError(readSize)) return readSize;
|
||||
|
||||
/* check result */
|
||||
if (tableLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
|
||||
|
||||
/* Prepare base value per rank */
|
||||
{ U32 n, nextRankStart = 0;
|
||||
for (n=1; n<=tableLog; n++) {
|
||||
U32 current = nextRankStart;
|
||||
nextRankStart += (rankVal[n] << (n-1));
|
||||
rankVal[n] = current;
|
||||
} }
|
||||
|
||||
/* fill nbBits */
|
||||
{ U32 n; for (n=0; n<nbSymbols; n++) {
|
||||
const U32 w = huffWeight[n];
|
||||
CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
|
||||
}}
|
||||
|
||||
/* fill val */
|
||||
{ U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0};
|
||||
U16 valPerRank[HUF_MAX_TABLELOG+1] = {0};
|
||||
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
|
||||
/* determine stating value per rank */
|
||||
{ U16 min = 0;
|
||||
U32 n; for (n=HUF_MAX_TABLELOG; n>0; n--) {
|
||||
valPerRank[n] = min; /* get starting value within each rank */
|
||||
min += nbPerRank[n];
|
||||
min >>= 1;
|
||||
} }
|
||||
/* assign value within rank, symbol order */
|
||||
{ U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; }
|
||||
}
|
||||
|
||||
return readSize;
|
||||
}
|
||||
|
||||
|
||||
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
||||
{
|
||||
const U32 largestBits = huffNode[lastNonNull].nbBits;
|
||||
if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */
|
||||
|
||||
/* there are several too large elements (at least >= 2) */
|
||||
{ int totalCost = 0;
|
||||
const U32 baseCost = 1 << (largestBits - maxNbBits);
|
||||
U32 n = lastNonNull;
|
||||
|
||||
while (huffNode[n].nbBits > maxNbBits) {
|
||||
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
|
||||
huffNode[n].nbBits = (BYTE)maxNbBits;
|
||||
n --;
|
||||
} /* n stops at huffNode[n].nbBits <= maxNbBits */
|
||||
while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */
|
||||
|
||||
/* renorm totalCost */
|
||||
totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */
|
||||
|
||||
/* repay normalized cost */
|
||||
{ U32 const noSymbol = 0xF0F0F0F0;
|
||||
U32 rankLast[HUF_MAX_TABLELOG+1];
|
||||
int pos;
|
||||
|
||||
/* Get pos of last (smallest) symbol per rank */
|
||||
memset(rankLast, 0xF0, sizeof(rankLast));
|
||||
{ U32 currentNbBits = maxNbBits;
|
||||
for (pos=n ; pos >= 0; pos--) {
|
||||
if (huffNode[pos].nbBits >= currentNbBits) continue;
|
||||
currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */
|
||||
rankLast[maxNbBits-currentNbBits] = pos;
|
||||
} }
|
||||
|
||||
while (totalCost > 0) {
|
||||
U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
|
||||
for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
|
||||
U32 highPos = rankLast[nBitsToDecrease];
|
||||
U32 lowPos = rankLast[nBitsToDecrease-1];
|
||||
if (highPos == noSymbol) continue;
|
||||
if (lowPos == noSymbol) break;
|
||||
{ U32 const highTotal = huffNode[highPos].count;
|
||||
U32 const lowTotal = 2 * huffNode[lowPos].count;
|
||||
if (highTotal <= lowTotal) break;
|
||||
} }
|
||||
/* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
|
||||
while ((nBitsToDecrease<=HUF_MAX_TABLELOG) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
|
||||
nBitsToDecrease ++;
|
||||
totalCost -= 1 << (nBitsToDecrease-1);
|
||||
if (rankLast[nBitsToDecrease-1] == noSymbol)
|
||||
rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */
|
||||
huffNode[rankLast[nBitsToDecrease]].nbBits ++;
|
||||
if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
|
||||
rankLast[nBitsToDecrease] = noSymbol;
|
||||
else {
|
||||
rankLast[nBitsToDecrease]--;
|
||||
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
|
||||
rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
|
||||
} } /* while (totalCost > 0) */
|
||||
|
||||
while (totalCost < 0) { /* Sometimes, cost correction overshoot */
|
||||
if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */
|
||||
while (huffNode[n].nbBits == maxNbBits) n--;
|
||||
huffNode[n+1].nbBits--;
|
||||
rankLast[1] = n+1;
|
||||
totalCost++;
|
||||
continue;
|
||||
}
|
||||
huffNode[ rankLast[1] + 1 ].nbBits--;
|
||||
rankLast[1]++;
|
||||
totalCost ++;
|
||||
} } } /* there are several too large elements (at least >= 2) */
|
||||
|
||||
return maxNbBits;
|
||||
}
|
||||
|
||||
|
||||
typedef struct {
|
||||
U32 base;
|
||||
U32 current;
|
||||
} rankPos;
|
||||
|
||||
static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
|
||||
{
|
||||
rankPos rank[32];
|
||||
U32 n;
|
||||
|
||||
memset(rank, 0, sizeof(rank));
|
||||
for (n=0; n<=maxSymbolValue; n++) {
|
||||
U32 r = BIT_highbit32(count[n] + 1);
|
||||
rank[r].base ++;
|
||||
}
|
||||
for (n=30; n>0; n--) rank[n-1].base += rank[n].base;
|
||||
for (n=0; n<32; n++) rank[n].current = rank[n].base;
|
||||
for (n=0; n<=maxSymbolValue; n++) {
|
||||
U32 const c = count[n];
|
||||
U32 const r = BIT_highbit32(c+1) + 1;
|
||||
U32 pos = rank[r].current++;
|
||||
while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--;
|
||||
huffNode[pos].count = c;
|
||||
huffNode[pos].byte = (BYTE)n;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#define STARTNODE (HUF_MAX_SYMBOL_VALUE+1)
|
||||
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
|
||||
{
|
||||
nodeElt huffNode0[2*HUF_MAX_SYMBOL_VALUE+1 +1];
|
||||
nodeElt* huffNode = huffNode0 + 1;
|
||||
U32 n, nonNullRank;
|
||||
int lowS, lowN;
|
||||
U16 nodeNb = STARTNODE;
|
||||
U32 nodeRoot;
|
||||
|
||||
/* safety checks */
|
||||
if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG;
|
||||
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return ERROR(GENERIC);
|
||||
memset(huffNode0, 0, sizeof(huffNode0));
|
||||
|
||||
/* sort, decreasing order */
|
||||
HUF_sort(huffNode, count, maxSymbolValue);
|
||||
|
||||
/* init for parents */
|
||||
nonNullRank = maxSymbolValue;
|
||||
while(huffNode[nonNullRank].count == 0) nonNullRank--;
|
||||
lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
|
||||
huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
|
||||
huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
|
||||
nodeNb++; lowS-=2;
|
||||
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
|
||||
huffNode0[0].count = (U32)(1U<<31);
|
||||
|
||||
/* create parents */
|
||||
while (nodeNb <= nodeRoot) {
|
||||
U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
|
||||
U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
|
||||
huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
|
||||
huffNode[n1].parent = huffNode[n2].parent = nodeNb;
|
||||
nodeNb++;
|
||||
}
|
||||
|
||||
/* distribute weights (unlimited tree height) */
|
||||
huffNode[nodeRoot].nbBits = 0;
|
||||
for (n=nodeRoot-1; n>=STARTNODE; n--)
|
||||
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
|
||||
for (n=0; n<=nonNullRank; n++)
|
||||
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
|
||||
|
||||
/* enforce maxTableLog */
|
||||
maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
|
||||
|
||||
/* fill result into tree (val, nbBits) */
|
||||
{ U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0};
|
||||
U16 valPerRank[HUF_MAX_TABLELOG+1] = {0};
|
||||
if (maxNbBits > HUF_MAX_TABLELOG) return ERROR(GENERIC); /* check fit into table */
|
||||
for (n=0; n<=nonNullRank; n++)
|
||||
nbPerRank[huffNode[n].nbBits]++;
|
||||
/* determine stating value per rank */
|
||||
{ U16 min = 0;
|
||||
for (n=maxNbBits; n>0; n--) {
|
||||
valPerRank[n] = min; /* get starting value within each rank */
|
||||
min += nbPerRank[n];
|
||||
min >>= 1;
|
||||
} }
|
||||
for (n=0; n<=maxSymbolValue; n++)
|
||||
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
|
||||
for (n=0; n<=maxSymbolValue; n++)
|
||||
tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
|
||||
}
|
||||
|
||||
return maxNbBits;
|
||||
}
|
||||
|
||||
static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
|
||||
{
|
||||
BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
|
||||
}
|
||||
|
||||
size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
|
||||
|
||||
#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
|
||||
|
||||
#define HUF_FLUSHBITS_1(stream) \
|
||||
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) HUF_FLUSHBITS(stream)
|
||||
|
||||
#define HUF_FLUSHBITS_2(stream) \
|
||||
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) HUF_FLUSHBITS(stream)
|
||||
|
||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
size_t n;
|
||||
const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
|
||||
BIT_CStream_t bitC;
|
||||
|
||||
/* init */
|
||||
if (dstSize < 8) return 0; /* not enough space to compress */
|
||||
{ size_t const errorCode = BIT_initCStream(&bitC, op, oend-op);
|
||||
if (HUF_isError(errorCode)) return 0; }
|
||||
|
||||
n = srcSize & ~3; /* join to mod 4 */
|
||||
switch (srcSize & 3)
|
||||
{
|
||||
case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable);
|
||||
HUF_FLUSHBITS_2(&bitC);
|
||||
case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
|
||||
HUF_FLUSHBITS_1(&bitC);
|
||||
case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
|
||||
HUF_FLUSHBITS(&bitC);
|
||||
case 0 :
|
||||
default: ;
|
||||
}
|
||||
|
||||
for (; n>0; n-=4) { /* note : n&3==0 at this stage */
|
||||
HUF_encodeSymbol(&bitC, ip[n- 1], CTable);
|
||||
HUF_FLUSHBITS_1(&bitC);
|
||||
HUF_encodeSymbol(&bitC, ip[n- 2], CTable);
|
||||
HUF_FLUSHBITS_2(&bitC);
|
||||
HUF_encodeSymbol(&bitC, ip[n- 3], CTable);
|
||||
HUF_FLUSHBITS_1(&bitC);
|
||||
HUF_encodeSymbol(&bitC, ip[n- 4], CTable);
|
||||
HUF_FLUSHBITS(&bitC);
|
||||
}
|
||||
|
||||
return BIT_closeCStream(&bitC);
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
||||
{
|
||||
size_t segmentSize = (srcSize+3)/4; /* first 3 segments */
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
const BYTE* const iend = ip + srcSize;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
size_t errorCode;
|
||||
|
||||
if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */
|
||||
if (srcSize < 12) return 0; /* no saving possible : too small input */
|
||||
op += 6; /* jumpTable */
|
||||
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
MEM_writeLE16(ostart, (U16)errorCode);
|
||||
|
||||
ip += segmentSize;
|
||||
op += errorCode;
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
MEM_writeLE16(ostart+2, (U16)errorCode);
|
||||
|
||||
ip += segmentSize;
|
||||
op += errorCode;
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
MEM_writeLE16(ostart+4, (U16)errorCode);
|
||||
|
||||
ip += segmentSize;
|
||||
op += errorCode;
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
|
||||
op += errorCode;
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
static size_t HUF_compress_internal (
|
||||
void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog,
|
||||
unsigned singleStream)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
|
||||
U32 count[HUF_MAX_SYMBOL_VALUE+1];
|
||||
HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1];
|
||||
size_t errorCode;
|
||||
|
||||
/* checks & inits */
|
||||
if (srcSize < 1) return 0; /* Uncompressed - note : 1 means rle, so first byte must be correct */
|
||||
if (dstSize < 1) return 0; /* not compressible within dst budget */
|
||||
if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */
|
||||
if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE;
|
||||
if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG;
|
||||
|
||||
/* Scan input and build symbol stats */
|
||||
errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }
|
||||
if (errorCode <= (srcSize >> 7)+1) return 0; /* Heuristic : not compressible enough */
|
||||
|
||||
/* Build Huffman Tree */
|
||||
errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
huffLog = (U32)errorCode;
|
||||
|
||||
/* Write table description header */
|
||||
errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode + 12 >= srcSize) return 0; /* not useful to try compression */
|
||||
op += errorCode;
|
||||
|
||||
/* Compress */
|
||||
if (singleStream)
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */
|
||||
else
|
||||
errorCode = HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
op += errorCode;
|
||||
|
||||
/* check compressibility */
|
||||
if ((size_t)(op-ostart) >= srcSize-1)
|
||||
return 0;
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_compress1X (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1);
|
||||
}
|
||||
|
||||
size_t HUF_compress2 (void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize,
|
||||
unsigned maxSymbolValue, unsigned huffLog)
|
||||
{
|
||||
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_DEFAULT_TABLELOG);
|
||||
}
|
||||
|
||||
|
||||
/* *******************************************************
|
||||
* Huff0 : Huffman block decompression
|
||||
* HUF : Huffman block decompression
|
||||
*********************************************************/
|
||||
typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */
|
||||
|
||||
@ -587,77 +84,6 @@ typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* doubl
|
||||
|
||||
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
|
||||
|
||||
/*! HUF_readStats() :
|
||||
Read compact Huffman tree, saved by HUF_writeCTable().
|
||||
`huffWeight` is destination buffer.
|
||||
@return : size read from `src`
|
||||
*/
|
||||
static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 weightTotal;
|
||||
U32 tableLog;
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
size_t iSize = ip[0];
|
||||
size_t oSize;
|
||||
|
||||
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
if (iSize >= 128) { /* special header */
|
||||
if (iSize >= (242)) { /* RLE */
|
||||
static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
|
||||
oSize = l[iSize-242];
|
||||
memset(huffWeight, 1, hwSize);
|
||||
iSize = 0;
|
||||
}
|
||||
else { /* Incompressible */
|
||||
oSize = iSize - 127;
|
||||
iSize = ((oSize+1)/2);
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
if (oSize >= hwSize) return ERROR(corruption_detected);
|
||||
ip += 1;
|
||||
{ U32 n;
|
||||
for (n=0; n<oSize; n+=2) {
|
||||
huffWeight[n] = ip[n/2] >> 4;
|
||||
huffWeight[n+1] = ip[n/2] & 15;
|
||||
} } } }
|
||||
else { /* header compressed with FSE (normal case) */
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
|
||||
if (FSE_isError(oSize)) return oSize;
|
||||
}
|
||||
|
||||
/* collect weight stats */
|
||||
memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32));
|
||||
weightTotal = 0;
|
||||
{ U32 n; for (n=0; n<oSize; n++) {
|
||||
if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
|
||||
rankStats[huffWeight[n]]++;
|
||||
weightTotal += (1 << huffWeight[n]) >> 1;
|
||||
}}
|
||||
|
||||
/* get last non-null symbol weight (implied, total must be 2^n) */
|
||||
tableLog = BIT_highbit32(weightTotal) + 1;
|
||||
if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
|
||||
/* determine last weight */
|
||||
{ U32 const total = 1 << tableLog;
|
||||
U32 const rest = total - weightTotal;
|
||||
U32 const verif = 1 << BIT_highbit32(rest);
|
||||
U32 const lastWeight = BIT_highbit32(rest) + 1;
|
||||
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
|
||||
huffWeight[oSize] = (BYTE)lastWeight;
|
||||
rankStats[lastWeight]++;
|
||||
}
|
||||
|
||||
/* check tree construction validity */
|
||||
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
|
||||
|
||||
/* results */
|
||||
*nbSymbolsPtr = (U32)(oSize+1);
|
||||
*tableLogPtr = tableLog;
|
||||
return iSize+1;
|
||||
}
|
||||
|
||||
|
||||
/*-***************************/
|
||||
@ -688,7 +114,7 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
|
||||
|
||||
/* Prepare ranks */
|
||||
nextRankStart = 0;
|
||||
for (n=1; n<=tableLog; n++) {
|
||||
for (n=1; n<tableLog+1; n++) {
|
||||
U32 current = nextRankStart;
|
||||
nextRankStart += (rankVal[n] << (n-1));
|
||||
rankVal[n] = current;
|
||||
@ -1016,7 +442,7 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
||||
|
||||
/* Get start index of each weight */
|
||||
{ U32 w, nextRankStart = 0;
|
||||
for (w=1; w<=maxW; w++) {
|
||||
for (w=1; w<maxW+1; w++) {
|
||||
U32 current = nextRankStart;
|
||||
nextRankStart += rankStats[w];
|
||||
rankStart[w] = current;
|
||||
@ -1028,8 +454,8 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
||||
/* sort symbols by weight */
|
||||
{ U32 s;
|
||||
for (s=0; s<nbSymbols; s++) {
|
||||
U32 w = weightList[s];
|
||||
U32 r = rankStart[w]++;
|
||||
U32 const w = weightList[s];
|
||||
U32 const r = rankStart[w]++;
|
||||
sortedSymbol[r].symbol = (BYTE)s;
|
||||
sortedSymbol[r].weight = (BYTE)w;
|
||||
}
|
||||
@ -1037,21 +463,23 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
||||
}
|
||||
|
||||
/* Build rankVal */
|
||||
{ const U32 minBits = tableLog+1 - maxW;
|
||||
{ U32* const rankVal0 = rankVal[0];
|
||||
{ int const rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
|
||||
U32 nextRankVal = 0;
|
||||
U32 w, consumed;
|
||||
const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
|
||||
U32* rankVal0 = rankVal[0];
|
||||
for (w=1; w<=maxW; w++) {
|
||||
U32 w;
|
||||
for (w=1; w<maxW+1; w++) {
|
||||
U32 current = nextRankVal;
|
||||
nextRankVal += rankStats[w] << (w+rescale);
|
||||
rankVal0[w] = current;
|
||||
}
|
||||
for (consumed = minBits; consumed <= memLog - minBits; consumed++) {
|
||||
U32* rankValPtr = rankVal[consumed];
|
||||
for (w = 1; w <= maxW; w++) {
|
||||
} }
|
||||
{ U32 const minBits = tableLog+1 - maxW;
|
||||
U32 consumed;
|
||||
for (consumed = minBits; consumed < memLog - minBits + 1; consumed++) {
|
||||
U32* const rankValPtr = rankVal[consumed];
|
||||
U32 w;
|
||||
for (w = 1; w < maxW+1; w++) {
|
||||
rankValPtr[w] = rankVal0[w] >> consumed;
|
||||
} } }
|
||||
} } } }
|
||||
|
||||
HUF_fillDTableX4(dt, memLog,
|
||||
sortedSymbol, sizeOfSort,
|
||||
@ -1362,7 +790,7 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
|
||||
|
||||
/* Get start index of each weight */
|
||||
{ U32 w, nextRankStart = 0;
|
||||
for (w=1; w<=maxW; w++) {
|
||||
for (w=1; w<maxW+1; w++) {
|
||||
U32 current = nextRankStart;
|
||||
nextRankStart += rankStats[w];
|
||||
rankStart[w] = current;
|
||||
@ -1388,14 +816,14 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
|
||||
U32 w, consumed;
|
||||
const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
|
||||
U32* rankVal0 = rankVal[0];
|
||||
for (w=1; w<=maxW; w++) {
|
||||
for (w=1; w<maxW+1; w++) {
|
||||
U32 current = nextRankVal;
|
||||
nextRankVal += rankStats[w] << (w+rescale);
|
||||
rankVal0[w] = current;
|
||||
}
|
||||
for (consumed = minBits; consumed <= memLog - minBits; consumed++) {
|
||||
U32* rankValPtr = rankVal[consumed];
|
||||
for (w = 1; w <= maxW; w++) {
|
||||
for (w = 1; w < maxW+1; w++) {
|
||||
rankValPtr[w] = rankVal0[w] >> consumed;
|
||||
} } }
|
||||
|
||||
@ -1421,17 +849,17 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
|
||||
|
||||
static U32 HUF_decodeSymbolX6(void* op, BIT_DStream_t* DStream, const HUF_DDescX6* dd, const HUF_DSeqX6* ds, const U32 dtLog)
|
||||
{
|
||||
const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
|
||||
size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
|
||||
memcpy(op, ds+val, sizeof(HUF_DSeqX6));
|
||||
BIT_skipBits(DStream, dd[val].nbBits);
|
||||
return dd[val].nbBytes;
|
||||
}
|
||||
|
||||
static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStream,
|
||||
static U32 HUF_decodeLastSymbolsX6(void* op, U32 const maxL, BIT_DStream_t* DStream,
|
||||
const HUF_DDescX6* dd, const HUF_DSeqX6* ds, const U32 dtLog)
|
||||
{
|
||||
const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
|
||||
U32 length = dd[val].nbBytes;
|
||||
size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
|
||||
U32 const length = dd[val].nbBytes;
|
||||
if (length <= maxL) {
|
||||
memcpy(op, ds+val, length);
|
||||
BIT_skipBits(DStream, dd[val].nbBits);
|
||||
@ -1484,7 +912,6 @@ static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c
|
||||
return p-pStart;
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_decompress1X6_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
@ -1493,17 +920,15 @@ size_t HUF_decompress1X6_usingDTable(
|
||||
const BYTE* const istart = (const BYTE*) cSrc;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
|
||||
const U32 dtLog = DTable[0];
|
||||
size_t errorCode;
|
||||
BIT_DStream_t bitD;
|
||||
|
||||
/* Init */
|
||||
BIT_DStream_t bitD;
|
||||
errorCode = BIT_initDStream(&bitD, istart, cSrcSize);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD, istart, cSrcSize);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
|
||||
/* finish bitStreams one by one */
|
||||
HUF_decodeStreamX6(ostart, &bitD, oend, DTable, dtLog);
|
||||
{ U32 const dtLog = DTable[0];
|
||||
HUF_decodeStreamX6(ostart, &bitD, oend, DTable, dtLog); }
|
||||
|
||||
/* check */
|
||||
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
|
||||
@ -1517,7 +942,7 @@ size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
||||
HUF_CREATE_STATIC_DTABLEX6(DTable, HUF_MAX_TABLELOG);
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t hSize = HUF_readDTableX6 (DTable, cSrc, cSrcSize);
|
||||
size_t const hSize = HUF_readDTableX6 (DTable, cSrc, cSrcSize);
|
||||
if (HUF_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize;
|
||||
@ -1527,6 +952,24 @@ size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
||||
}
|
||||
|
||||
|
||||
#define HUF_DECODE_ROUNDX6 \
|
||||
HUF_DECODE_SYMBOLX6_2(op1, &bitD1); \
|
||||
HUF_DECODE_SYMBOLX6_2(op2, &bitD2); \
|
||||
HUF_DECODE_SYMBOLX6_2(op3, &bitD3); \
|
||||
HUF_DECODE_SYMBOLX6_2(op4, &bitD4); \
|
||||
HUF_DECODE_SYMBOLX6_1(op1, &bitD1); \
|
||||
HUF_DECODE_SYMBOLX6_1(op2, &bitD2); \
|
||||
HUF_DECODE_SYMBOLX6_1(op3, &bitD3); \
|
||||
HUF_DECODE_SYMBOLX6_1(op4, &bitD4); \
|
||||
HUF_DECODE_SYMBOLX6_2(op1, &bitD1); \
|
||||
HUF_DECODE_SYMBOLX6_2(op2, &bitD2); \
|
||||
HUF_DECODE_SYMBOLX6_2(op3, &bitD3); \
|
||||
HUF_DECODE_SYMBOLX6_2(op4, &bitD4); \
|
||||
HUF_DECODE_SYMBOLX6_0(op1, &bitD1); \
|
||||
HUF_DECODE_SYMBOLX6_0(op2, &bitD2); \
|
||||
HUF_DECODE_SYMBOLX6_0(op3, &bitD3); \
|
||||
HUF_DECODE_SYMBOLX6_0(op4, &bitD4);
|
||||
|
||||
size_t HUF_decompress4X6_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
@ -1534,6 +977,7 @@ size_t HUF_decompress4X6_usingDTable(
|
||||
{
|
||||
/* Check */
|
||||
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
|
||||
if (dstSize < 64) return ERROR(dstSize_tooSmall); /* only work for dstSize >= 64 */
|
||||
|
||||
{ const BYTE* const istart = (const BYTE*) cSrc;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
@ -1544,7 +988,6 @@ size_t HUF_decompress4X6_usingDTable(
|
||||
const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
|
||||
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
|
||||
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
|
||||
size_t errorCode;
|
||||
|
||||
/* Init */
|
||||
BIT_DStream_t bitD1;
|
||||
@ -1571,43 +1014,41 @@ size_t HUF_decompress4X6_usingDTable(
|
||||
|
||||
length4 = cSrcSize - (length1 + length2 + length3 + 6);
|
||||
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
|
||||
errorCode = BIT_initDStream(&bitD1, istart1, length1);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
errorCode = BIT_initDStream(&bitD2, istart2, length2);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
errorCode = BIT_initDStream(&bitD3, istart3, length3);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
errorCode = BIT_initDStream(&bitD4, istart4, length4);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
|
||||
/* 16-64 symbols per loop (4-16 symbols per stream) */
|
||||
/* 4-64 symbols per loop (1-16 symbols per stream) */
|
||||
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
|
||||
if (endSignal==BIT_DStream_unfinished) {
|
||||
HUF_DECODE_ROUNDX6;
|
||||
if (sizeof(bitD1.bitContainer)==4) { /* need to decode at least 4 bytes per stream */
|
||||
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
|
||||
HUF_DECODE_ROUNDX6;
|
||||
}
|
||||
{ U32 const saved2 = MEM_read32(opStart2); /* saved from overwrite */
|
||||
U32 const saved3 = MEM_read32(opStart3);
|
||||
U32 const saved4 = MEM_read32(opStart4);
|
||||
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
|
||||
for ( ; (op3 <= opStart4) && (endSignal==BIT_DStream_unfinished) && (op4<=(oend-16)) ; ) {
|
||||
HUF_DECODE_SYMBOLX6_2(op1, &bitD1);
|
||||
HUF_DECODE_SYMBOLX6_2(op2, &bitD2);
|
||||
HUF_DECODE_SYMBOLX6_2(op3, &bitD3);
|
||||
HUF_DECODE_SYMBOLX6_2(op4, &bitD4);
|
||||
HUF_DECODE_SYMBOLX6_1(op1, &bitD1);
|
||||
HUF_DECODE_SYMBOLX6_1(op2, &bitD2);
|
||||
HUF_DECODE_SYMBOLX6_1(op3, &bitD3);
|
||||
HUF_DECODE_SYMBOLX6_1(op4, &bitD4);
|
||||
HUF_DECODE_SYMBOLX6_2(op1, &bitD1);
|
||||
HUF_DECODE_SYMBOLX6_2(op2, &bitD2);
|
||||
HUF_DECODE_SYMBOLX6_2(op3, &bitD3);
|
||||
HUF_DECODE_SYMBOLX6_2(op4, &bitD4);
|
||||
HUF_DECODE_SYMBOLX6_0(op1, &bitD1);
|
||||
HUF_DECODE_SYMBOLX6_0(op2, &bitD2);
|
||||
HUF_DECODE_SYMBOLX6_0(op3, &bitD3);
|
||||
HUF_DECODE_SYMBOLX6_0(op4, &bitD4);
|
||||
|
||||
HUF_DECODE_ROUNDX6;
|
||||
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
|
||||
}
|
||||
MEM_write32(opStart2, saved2);
|
||||
MEM_write32(opStart3, saved3);
|
||||
MEM_write32(opStart4, saved4);
|
||||
} }
|
||||
|
||||
/* check corruption */
|
||||
if (op1 > opStart2) return ERROR(corruption_detected);
|
||||
if (op2 > opStart3) return ERROR(corruption_detected);
|
||||
if (op3 > opStart4) return ERROR(corruption_detected);
|
||||
/* note : op4 supposed already verified within main loop */
|
||||
/* note : op4 already verified within main loop */
|
||||
|
||||
/* finish bitStreams one by one */
|
||||
HUF_decodeStreamX6(op1, &bitD1, opStart2, DTable, dtLog);
|
||||
@ -1671,12 +1112,7 @@ typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc,
|
||||
size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
static const decompressionAlgo decompress[3] = { HUF_decompress4X2, HUF_decompress4X4, HUF_decompress4X6 };
|
||||
/* estimate decompression time */
|
||||
U32 Q;
|
||||
const U32 D256 = (U32)(dstSize >> 8);
|
||||
U32 Dtime[3];
|
||||
U32 algoNb = 0;
|
||||
int n;
|
||||
U32 Dtime[3]; /* decompression time estimation */
|
||||
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
@ -1685,16 +1121,19 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
/* decoder timing evaluation */
|
||||
Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
|
||||
for (n=0; n<3; n++)
|
||||
{ U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
|
||||
U32 const D256 = (U32)(dstSize >> 8);
|
||||
U32 n; for (n=0; n<3; n++)
|
||||
Dtime[n] = algoTime[Q][n].tableTime + (algoTime[Q][n].decode256Time * D256);
|
||||
}
|
||||
|
||||
Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */
|
||||
|
||||
{ U32 algoNb = 0;
|
||||
if (Dtime[1] < Dtime[0]) algoNb = 1;
|
||||
if (Dtime[2] < Dtime[algoNb]) algoNb = 2;
|
||||
|
||||
// if (Dtime[2] < Dtime[algoNb]) algoNb = 2; /* current speed of HUF_decompress4X6 is not good */
|
||||
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
//return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
|
||||
//return HUF_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
|
||||
@ -1,5 +1,5 @@
|
||||
/* ******************************************************************
|
||||
Huff0 : Huffman codec, part of New Generation Entropy library
|
||||
Huffman codec, part of New Generation Entropy library
|
||||
header file, for static linking only
|
||||
Copyright (C) 2013-2016, Yann Collet
|
||||
|
||||
@ -31,8 +31,8 @@
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef HUFF0_STATIC_H
|
||||
#define HUFF0_STATIC_H
|
||||
#ifndef HUF_STATIC_H
|
||||
#define HUF_STATIC_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
@ -42,24 +42,26 @@ extern "C" {
|
||||
/* ****************************************
|
||||
* Dependency
|
||||
******************************************/
|
||||
#include "huff0.h"
|
||||
#include "huf.h"
|
||||
#include "fse.h"
|
||||
#include "bitstream.h"
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Static allocation
|
||||
******************************************/
|
||||
/* Huff0 buffer bounds */
|
||||
/* HUF buffer bounds */
|
||||
#define HUF_CTABLEBOUND 129
|
||||
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
|
||||
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* static allocation of Huff0's Compression Table */
|
||||
/* static allocation of HUF's Compression Table */
|
||||
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
|
||||
U32 name##hb[maxSymbolValue+1]; \
|
||||
void* name##hv = &(name##hb); \
|
||||
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
|
||||
|
||||
/* static allocation of Huff0's DTable */
|
||||
/* static allocation of HUF's DTable */
|
||||
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<maxTableLog))
|
||||
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
||||
unsigned short DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
|
||||
@ -74,11 +76,11 @@ extern "C" {
|
||||
******************************************/
|
||||
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
|
||||
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder */
|
||||
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Huff0 detailed API
|
||||
* HUF detailed API
|
||||
******************************************/
|
||||
/*!
|
||||
HUF_compress() does the following:
|
||||
@ -120,7 +122,7 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
|
||||
|
||||
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
||||
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbol decoder */
|
||||
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
|
||||
|
||||
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
|
||||
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
||||
@ -132,8 +134,94 @@ size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* c
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Constants
|
||||
****************************************************************/
|
||||
#define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
|
||||
#define HUF_MAX_TABLELOG 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
|
||||
#define HUF_DEFAULT_TABLELOG HUF_MAX_TABLELOG /* tableLog by default, when not specified */
|
||||
#define HUF_MAX_SYMBOL_VALUE 255
|
||||
#if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG)
|
||||
# error "HUF_MAX_TABLELOG is too large !"
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*! HUF_readStats() :
|
||||
Read compact Huffman tree, saved by HUF_writeCTable().
|
||||
`huffWeight` is destination buffer.
|
||||
@return : size read from `src`
|
||||
*/
|
||||
MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 weightTotal;
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
size_t iSize = ip[0];
|
||||
size_t oSize;
|
||||
|
||||
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
if (iSize >= 128) { /* special header */
|
||||
if (iSize >= (242)) { /* RLE */
|
||||
static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
|
||||
oSize = l[iSize-242];
|
||||
memset(huffWeight, 1, hwSize);
|
||||
iSize = 0;
|
||||
}
|
||||
else { /* Incompressible */
|
||||
oSize = iSize - 127;
|
||||
iSize = ((oSize+1)/2);
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
if (oSize >= hwSize) return ERROR(corruption_detected);
|
||||
ip += 1;
|
||||
{ U32 n;
|
||||
for (n=0; n<oSize; n+=2) {
|
||||
huffWeight[n] = ip[n/2] >> 4;
|
||||
huffWeight[n+1] = ip[n/2] & 15;
|
||||
} } } }
|
||||
else { /* header compressed with FSE (normal case) */
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
|
||||
if (FSE_isError(oSize)) return oSize;
|
||||
}
|
||||
|
||||
/* collect weight stats */
|
||||
memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32));
|
||||
weightTotal = 0;
|
||||
{ U32 n; for (n=0; n<oSize; n++) {
|
||||
if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
|
||||
rankStats[huffWeight[n]]++;
|
||||
weightTotal += (1 << huffWeight[n]) >> 1;
|
||||
} }
|
||||
|
||||
/* get last non-null symbol weight (implied, total must be 2^n) */
|
||||
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
|
||||
if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
|
||||
*tableLogPtr = tableLog;
|
||||
/* determine last weight */
|
||||
{ U32 const total = 1 << tableLog;
|
||||
U32 const rest = total - weightTotal;
|
||||
U32 const verif = 1 << BIT_highbit32(rest);
|
||||
U32 const lastWeight = BIT_highbit32(rest) + 1;
|
||||
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
|
||||
huffWeight[oSize] = (BYTE)lastWeight;
|
||||
rankStats[lastWeight]++;
|
||||
} }
|
||||
|
||||
/* check tree construction validity */
|
||||
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
|
||||
|
||||
/* results */
|
||||
*nbSymbolsPtr = (U32)(oSize+1);
|
||||
return iSize+1;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* HUFF0_STATIC_H */
|
||||
#endif /* HUF_STATIC_H */
|
||||
@ -63,7 +63,7 @@ extern "C" {
|
||||
/*-**************************************************************
|
||||
* Basic Types
|
||||
*****************************************************************/
|
||||
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
@ -108,8 +108,8 @@ extern "C" {
|
||||
# endif
|
||||
#endif
|
||||
|
||||
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
|
||||
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
|
||||
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
|
||||
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
|
||||
|
||||
MEM_STATIC unsigned MEM_isLittleEndian(void)
|
||||
{
|
||||
@ -187,6 +187,48 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value)
|
||||
|
||||
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
||||
|
||||
MEM_STATIC U32 MEM_swap32(U32 in)
|
||||
{
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
return _byteswap_ulong(in);
|
||||
#elif defined (__GNUC__)
|
||||
return __builtin_bswap32(in);
|
||||
#else
|
||||
return ((in << 24) & 0xff000000 ) |
|
||||
((in << 8) & 0x00ff0000 ) |
|
||||
((in >> 8) & 0x0000ff00 ) |
|
||||
((in >> 24) & 0x000000ff );
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_swap64(U64 in)
|
||||
{
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
return _byteswap_uint64(in);
|
||||
#elif defined (__GNUC__)
|
||||
return __builtin_bswap64(in);
|
||||
#else
|
||||
return ((in << 56) & 0xff00000000000000ULL) |
|
||||
((in << 40) & 0x00ff000000000000ULL) |
|
||||
((in << 24) & 0x0000ff0000000000ULL) |
|
||||
((in << 8) & 0x000000ff00000000ULL) |
|
||||
((in >> 8) & 0x00000000ff000000ULL) |
|
||||
((in >> 24) & 0x0000000000ff0000ULL) |
|
||||
((in >> 40) & 0x000000000000ff00ULL) |
|
||||
((in >> 56) & 0x00000000000000ffULL);
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_swapST(size_t in)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
return (size_t)MEM_swap32((U32)in);
|
||||
else
|
||||
return (size_t)MEM_swap64((U64)in);
|
||||
}
|
||||
|
||||
/*=== Little endian r/w ===*/
|
||||
|
||||
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
@ -212,51 +254,32 @@ MEM_STATIC U32 MEM_readLE32(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_read32(memPtr);
|
||||
else {
|
||||
const BYTE* p = (const BYTE*)memPtr;
|
||||
return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
|
||||
}
|
||||
else
|
||||
return MEM_swap32(MEM_read32(memPtr));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
|
||||
{
|
||||
if (MEM_isLittleEndian()) {
|
||||
if (MEM_isLittleEndian())
|
||||
MEM_write32(memPtr, val32);
|
||||
} else {
|
||||
BYTE* p = (BYTE*)memPtr;
|
||||
p[0] = (BYTE)val32;
|
||||
p[1] = (BYTE)(val32>>8);
|
||||
p[2] = (BYTE)(val32>>16);
|
||||
p[3] = (BYTE)(val32>>24);
|
||||
}
|
||||
else
|
||||
MEM_write32(memPtr, MEM_swap32(val32));
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_read64(memPtr);
|
||||
else {
|
||||
const BYTE* p = (const BYTE*)memPtr;
|
||||
return (U64)((U64)p[0] + ((U64)p[1]<<8) + ((U64)p[2]<<16) + ((U64)p[3]<<24)
|
||||
+ ((U64)p[4]<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56));
|
||||
}
|
||||
else
|
||||
return MEM_swap64(MEM_read64(memPtr));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
|
||||
{
|
||||
if (MEM_isLittleEndian()) {
|
||||
if (MEM_isLittleEndian())
|
||||
MEM_write64(memPtr, val64);
|
||||
} else {
|
||||
BYTE* p = (BYTE*)memPtr;
|
||||
p[0] = (BYTE)val64;
|
||||
p[1] = (BYTE)(val64>>8);
|
||||
p[2] = (BYTE)(val64>>16);
|
||||
p[3] = (BYTE)(val64>>24);
|
||||
p[4] = (BYTE)(val64>>32);
|
||||
p[5] = (BYTE)(val64>>40);
|
||||
p[6] = (BYTE)(val64>>48);
|
||||
p[7] = (BYTE)(val64>>56);
|
||||
}
|
||||
else
|
||||
MEM_write64(memPtr, MEM_swap64(val64));
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
||||
@ -275,6 +298,57 @@ MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
|
||||
MEM_writeLE64(memPtr, (U64)val);
|
||||
}
|
||||
|
||||
/*=== Big endian r/w ===*/
|
||||
|
||||
MEM_STATIC U32 MEM_readBE32(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_swap32(MEM_read32(memPtr));
|
||||
else
|
||||
return MEM_read32(memPtr);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
MEM_write32(memPtr, MEM_swap32(val32));
|
||||
else
|
||||
MEM_write32(memPtr, val32);
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_readBE64(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_swap64(MEM_read64(memPtr));
|
||||
else
|
||||
return MEM_read64(memPtr);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
MEM_write64(memPtr, MEM_swap64(val64));
|
||||
else
|
||||
MEM_write64(memPtr, val64);
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_readBEST(const void* memPtr)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
return (size_t)MEM_readBE32(memPtr);
|
||||
else
|
||||
return (size_t)MEM_readBE64(memPtr);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
MEM_writeBE32(memPtr, (U32)val);
|
||||
else
|
||||
MEM_writeBE64(memPtr, (U64)val);
|
||||
}
|
||||
|
||||
|
||||
/* function safe only for comparisons */
|
||||
MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
|
||||
{
|
||||
|
||||
167
contrib/libzstd/include/zstd/zbuff.h
Normal file
167
contrib/libzstd/include/zstd/zbuff.h
Normal file
@ -0,0 +1,167 @@
|
||||
/*
|
||||
Buffered version of Zstd compression library
|
||||
Copyright (C) 2015-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net/
|
||||
*/
|
||||
#ifndef ZSTD_BUFFERED_H
|
||||
#define ZSTD_BUFFERED_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* *************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
* Compiler specifics
|
||||
*****************************************************************/
|
||||
/*!
|
||||
* ZSTD_DLL_EXPORT :
|
||||
* Enable exporting of functions when building a Windows DLL
|
||||
*/
|
||||
#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
|
||||
# define ZSTDLIB_API __declspec(dllexport)
|
||||
#else
|
||||
# define ZSTDLIB_API
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Streaming functions
|
||||
***************************************/
|
||||
typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
|
||||
ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void);
|
||||
ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
|
||||
|
||||
ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
|
||||
ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
|
||||
ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
|
||||
ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
|
||||
|
||||
/*-*************************************************
|
||||
* Streaming compression - howto
|
||||
*
|
||||
* A ZBUFF_CCtx object is required to track streaming operation.
|
||||
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
|
||||
* ZBUFF_CCtx objects can be reused multiple times.
|
||||
*
|
||||
* Start by initializing ZBUF_CCtx.
|
||||
* Use ZBUFF_compressInit() to start a new compression operation.
|
||||
* Use ZBUFF_compressInitDictionary() for a compression which requires a dictionary.
|
||||
*
|
||||
* Use ZBUFF_compressContinue() repetitively to consume input stream.
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data.
|
||||
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst .
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush().
|
||||
* The nb of bytes written into `dst` will be reported into *dstCapacityPtr.
|
||||
* Note that the function cannot output more than *dstCapacityPtr,
|
||||
* therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* ZBUFF_compressEnd() instructs to finish a frame.
|
||||
* It will perform a flush and write frame epilogue.
|
||||
* The epilogue is required for decoders to consider a frame completed.
|
||||
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
|
||||
* In which case, call again ZBUFF_compressFlush() to complete the flush.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedCInSize / ZBUFF_recommendedCOutSize
|
||||
* input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, it improves latency to use this value (skipped buffering).
|
||||
* output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering.
|
||||
* By using both, it ensures that input will be entirely consumed, and output will always contain the result, reducing intermediate buffering.
|
||||
* **************************************************/
|
||||
|
||||
|
||||
typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
|
||||
ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void);
|
||||
ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
|
||||
|
||||
ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
|
||||
ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
|
||||
void* dst, size_t* dstCapacityPtr,
|
||||
const void* src, size_t* srcSizePtr);
|
||||
|
||||
/*-***************************************************************************
|
||||
* Streaming decompression howto
|
||||
*
|
||||
* A ZBUFF_DCtx object is required to track streaming operations.
|
||||
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
|
||||
* Use ZBUFF_decompressInit() to start a new decompression operation,
|
||||
* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
|
||||
* Note that ZBUFF_DCtx objects can be re-init multiple times.
|
||||
*
|
||||
* Use ZBUFF_decompressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
||||
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
|
||||
* or 0 when a frame is completely decoded,
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
|
||||
* output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFF_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* *******************************************************************************/
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Tool functions
|
||||
***************************************/
|
||||
ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode);
|
||||
ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode);
|
||||
|
||||
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
|
||||
* These sizes are just hints, they tend to offer better latency */
|
||||
ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void);
|
||||
ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void);
|
||||
ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void);
|
||||
ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_BUFFERED_H */
|
||||
291
contrib/libzstd/include/zstd/zbuff_compress.c
Normal file
291
contrib/libzstd/include/zstd/zbuff_compress.c
Normal file
@ -0,0 +1,291 @@
|
||||
/*
|
||||
Buffered version of Zstd compression library
|
||||
Copyright (C) 2015-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net/
|
||||
*/
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stdlib.h>
|
||||
#include "error_private.h"
|
||||
#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */
|
||||
#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */
|
||||
#include "zbuff_static.h"
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
|
||||
|
||||
|
||||
/*_**************************************************
|
||||
* Streaming compression
|
||||
*
|
||||
* A ZBUFF_CCtx object is required to track streaming operation.
|
||||
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
|
||||
* Use ZBUFF_compressInit() to start a new compression operation.
|
||||
* ZBUFF_CCtx objects can be reused multiple times.
|
||||
*
|
||||
* Use ZBUFF_compressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
|
||||
* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst .
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
|
||||
* Note that it will not output more than *dstCapacityPtr.
|
||||
* Therefore, some content might still be left into its internal buffer if dst buffer is too small.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* ZBUFF_compressEnd() instructs to finish a frame.
|
||||
* It will perform a flush and write frame epilogue.
|
||||
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory)
|
||||
* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value.
|
||||
* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
|
||||
* **************************************************/
|
||||
|
||||
typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
|
||||
|
||||
/* *** Ressources *** */
|
||||
struct ZBUFF_CCtx_s {
|
||||
ZSTD_CCtx* zc;
|
||||
char* inBuff;
|
||||
size_t inBuffSize;
|
||||
size_t inToCompress;
|
||||
size_t inBuffPos;
|
||||
size_t inBuffTarget;
|
||||
size_t blockSize;
|
||||
char* outBuff;
|
||||
size_t outBuffSize;
|
||||
size_t outBuffContentSize;
|
||||
size_t outBuffFlushedSize;
|
||||
ZBUFF_cStage stage;
|
||||
}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
|
||||
|
||||
ZBUFF_CCtx* ZBUFF_createCCtx(void)
|
||||
{
|
||||
ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx));
|
||||
if (zbc==NULL) return NULL;
|
||||
memset(zbc, 0, sizeof(*zbc));
|
||||
zbc->zc = ZSTD_createCCtx();
|
||||
return zbc;
|
||||
}
|
||||
|
||||
size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
|
||||
{
|
||||
if (zbc==NULL) return 0; /* support free on NULL */
|
||||
ZSTD_freeCCtx(zbc->zc);
|
||||
free(zbc->inBuff);
|
||||
free(zbc->outBuff);
|
||||
free(zbc);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/* *** Initialization *** */
|
||||
|
||||
size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize)
|
||||
{
|
||||
/* allocate buffers */
|
||||
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
|
||||
if (zbc->inBuffSize < neededInBuffSize) {
|
||||
zbc->inBuffSize = neededInBuffSize;
|
||||
free(zbc->inBuff); /* should not be necessary */
|
||||
zbc->inBuff = (char*)malloc(neededInBuffSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2);
|
||||
}
|
||||
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
|
||||
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
|
||||
free(zbc->outBuff); /* should not be necessary */
|
||||
zbc->outBuff = (char*)malloc(zbc->outBuffSize);
|
||||
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
|
||||
{ size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize);
|
||||
if (ZSTD_isError(errorCode)) return errorCode; }
|
||||
|
||||
zbc->inToCompress = 0;
|
||||
zbc->inBuffPos = 0;
|
||||
zbc->inBuffTarget = zbc->blockSize;
|
||||
zbc->outBuffFlushedSize = 0;
|
||||
zbc->stage = ZBUFFcs_load;
|
||||
return 0; /* ready to go */
|
||||
}
|
||||
|
||||
|
||||
size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
params.fParams.contentSizeFlag = 0;
|
||||
ZSTD_adjustCParams(¶ms.cParams, 0, dictSize);
|
||||
return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
|
||||
}
|
||||
|
||||
size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
|
||||
{
|
||||
return ZBUFF_compressInitDictionary(zbc, NULL, 0, compressionLevel);
|
||||
}
|
||||
|
||||
|
||||
/* *** Compression *** */
|
||||
|
||||
static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
void* dst, size_t* dstCapacityPtr,
|
||||
const void* src, size_t* srcSizePtr,
|
||||
int flush) /* aggregate : wait for full block before compressing */
|
||||
{
|
||||
U32 notDone = 1;
|
||||
const char* const istart = (const char*)src;
|
||||
const char* const iend = istart + *srcSizePtr;
|
||||
const char* ip = istart;
|
||||
char* const ostart = (char*)dst;
|
||||
char* const oend = ostart + *dstCapacityPtr;
|
||||
char* op = ostart;
|
||||
|
||||
while (notDone) {
|
||||
switch(zbc->stage)
|
||||
{
|
||||
case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
|
||||
|
||||
case ZBUFFcs_load:
|
||||
/* complete inBuffer */
|
||||
{ size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
|
||||
zbc->inBuffPos += loaded;
|
||||
ip += loaded;
|
||||
if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) {
|
||||
notDone = 0; break; /* not enough input to get a full block : stop there, wait for more */
|
||||
} }
|
||||
/* compress current block (note : this stage cannot be stopped in the middle) */
|
||||
{ void* cDst;
|
||||
size_t cSize;
|
||||
size_t const iSize = zbc->inBuffPos - zbc->inToCompress;
|
||||
size_t oSize = oend-op;
|
||||
if (oSize >= ZSTD_compressBound(iSize))
|
||||
cDst = op; /* compress directly into output buffer (avoid flush stage) */
|
||||
else
|
||||
cDst = zbc->outBuff, oSize = zbc->outBuffSize;
|
||||
cSize = ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize);
|
||||
if (ZSTD_isError(cSize)) return cSize;
|
||||
/* prepare next block */
|
||||
zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;
|
||||
if (zbc->inBuffTarget > zbc->inBuffSize)
|
||||
zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize; /* note : inBuffSize >= blockSize */
|
||||
zbc->inToCompress = zbc->inBuffPos;
|
||||
if (cDst == op) { op += cSize; break; } /* no need to flush */
|
||||
zbc->outBuffContentSize = cSize;
|
||||
zbc->outBuffFlushedSize = 0;
|
||||
zbc->stage = ZBUFFcs_flush; /* continue to flush stage */
|
||||
}
|
||||
|
||||
case ZBUFFcs_flush:
|
||||
/* flush into dst */
|
||||
{ size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
|
||||
size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
|
||||
op += flushed;
|
||||
zbc->outBuffFlushedSize += flushed;
|
||||
if (toFlush!=flushed) { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */
|
||||
zbc->outBuffContentSize = 0;
|
||||
zbc->outBuffFlushedSize = 0;
|
||||
zbc->stage = ZBUFFcs_load;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
return ERROR(GENERIC); /* impossible */
|
||||
}
|
||||
}
|
||||
|
||||
*srcSizePtr = ip - istart;
|
||||
*dstCapacityPtr = op - ostart;
|
||||
{ size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
if (hintInSize==0) hintInSize = zbc->blockSize;
|
||||
return hintInSize;
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
|
||||
void* dst, size_t* dstCapacityPtr,
|
||||
const void* src, size_t* srcSizePtr)
|
||||
{
|
||||
return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* *** Finalize *** */
|
||||
|
||||
size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
|
||||
{
|
||||
size_t srcSize = 0;
|
||||
ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL */
|
||||
return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
|
||||
}
|
||||
|
||||
|
||||
size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + *dstCapacityPtr;
|
||||
BYTE* op = ostart;
|
||||
size_t outSize = *dstCapacityPtr;
|
||||
size_t epilogueSize, remaining;
|
||||
ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */
|
||||
op += outSize;
|
||||
epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize); /* epilogue into outBuff */
|
||||
zbc->outBuffContentSize += epilogueSize;
|
||||
outSize = oend-op;
|
||||
zbc->stage = ZBUFFcs_flush;
|
||||
remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */
|
||||
op += outSize;
|
||||
if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */
|
||||
*dstCapacityPtr = op-ostart; /* tells how many bytes were written */
|
||||
return remaining;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Tool functions
|
||||
***************************************/
|
||||
size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_MAX; }
|
||||
size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; }
|
||||
265
contrib/libzstd/include/zstd/zbuff_decompress.c
Normal file
265
contrib/libzstd/include/zstd/zbuff_decompress.c
Normal file
@ -0,0 +1,265 @@
|
||||
/*
|
||||
Buffered version of Zstd compression library
|
||||
Copyright (C) 2015-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net/
|
||||
*/
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stdlib.h>
|
||||
#include "error_private.h"
|
||||
#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */
|
||||
#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */
|
||||
#include "zbuff_static.h"
|
||||
|
||||
|
||||
/*-***************************************************************************
|
||||
* Streaming decompression howto
|
||||
*
|
||||
* A ZBUFF_DCtx object is required to track streaming operations.
|
||||
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
|
||||
* Use ZBUFF_decompressInit() to start a new decompression operation,
|
||||
* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
|
||||
* Note that ZBUFF_DCtx objects can be re-init multiple times.
|
||||
*
|
||||
* Use ZBUFF_decompressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
||||
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
|
||||
* or 0 when a frame is completely decoded,
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
|
||||
* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFF_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* *******************************************************************************/
|
||||
|
||||
typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
|
||||
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
|
||||
|
||||
/* *** Resource management *** */
|
||||
struct ZBUFF_DCtx_s {
|
||||
ZSTD_DCtx* zd;
|
||||
ZSTD_frameParams fParams;
|
||||
ZBUFF_dStage stage;
|
||||
char* inBuff;
|
||||
size_t inBuffSize;
|
||||
size_t inPos;
|
||||
char* outBuff;
|
||||
size_t outBuffSize;
|
||||
size_t outStart;
|
||||
size_t outEnd;
|
||||
size_t blockSize;
|
||||
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||
size_t lhSize;
|
||||
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
|
||||
|
||||
|
||||
ZBUFF_DCtx* ZBUFF_createDCtx(void)
|
||||
{
|
||||
ZBUFF_DCtx* zbd = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx));
|
||||
if (zbd==NULL) return NULL;
|
||||
memset(zbd, 0, sizeof(*zbd));
|
||||
zbd->zd = ZSTD_createDCtx();
|
||||
zbd->stage = ZBUFFds_init;
|
||||
return zbd;
|
||||
}
|
||||
|
||||
size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
|
||||
{
|
||||
if (zbd==NULL) return 0; /* support free on null */
|
||||
ZSTD_freeDCtx(zbd->zd);
|
||||
free(zbd->inBuff);
|
||||
free(zbd->outBuff);
|
||||
free(zbd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/* *** Initialization *** */
|
||||
|
||||
size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
|
||||
{
|
||||
zbd->stage = ZBUFFds_loadHeader;
|
||||
zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
|
||||
return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize);
|
||||
}
|
||||
|
||||
size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd)
|
||||
{
|
||||
return ZBUFF_decompressInitDictionary(zbd, NULL, 0);
|
||||
}
|
||||
|
||||
|
||||
/* *** Decompression *** */
|
||||
|
||||
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
||||
void* dst, size_t* dstCapacityPtr,
|
||||
const void* src, size_t* srcSizePtr)
|
||||
{
|
||||
const char* const istart = (const char*)src;
|
||||
const char* const iend = istart + *srcSizePtr;
|
||||
const char* ip = istart;
|
||||
char* const ostart = (char*)dst;
|
||||
char* const oend = ostart + *dstCapacityPtr;
|
||||
char* op = ostart;
|
||||
U32 notDone = 1;
|
||||
|
||||
while (notDone) {
|
||||
switch(zbd->stage)
|
||||
{
|
||||
case ZBUFFds_init :
|
||||
return ERROR(init_missing);
|
||||
|
||||
case ZBUFFds_loadHeader :
|
||||
{ size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
|
||||
if (hSize != 0) {
|
||||
size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */
|
||||
if (ZSTD_isError(hSize)) return hSize;
|
||||
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
||||
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
|
||||
zbd->lhSize += iend-ip; ip = iend; notDone = 0;
|
||||
*dstCapacityPtr = 0;
|
||||
return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
|
||||
}
|
||||
memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
|
||||
break;
|
||||
} }
|
||||
|
||||
/* Consume header */
|
||||
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */
|
||||
size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
|
||||
if (ZSTD_isError(h1Result)) return h1Result;
|
||||
if (h1Size < zbd->lhSize) { /* long header */
|
||||
size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);
|
||||
size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
|
||||
if (ZSTD_isError(h2Result)) return h2Result;
|
||||
} }
|
||||
|
||||
/* Frame header instruct buffer sizes */
|
||||
{ size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX);
|
||||
zbd->blockSize = blockSize;
|
||||
if (zbd->inBuffSize < blockSize) {
|
||||
free(zbd->inBuff);
|
||||
zbd->inBuffSize = blockSize;
|
||||
zbd->inBuff = (char*)malloc(blockSize);
|
||||
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
{ size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize;
|
||||
if (zbd->outBuffSize < neededOutSize) {
|
||||
free(zbd->outBuff);
|
||||
zbd->outBuffSize = neededOutSize;
|
||||
zbd->outBuff = (char*)malloc(neededOutSize);
|
||||
if (zbd->outBuff == NULL) return ERROR(memory_allocation);
|
||||
} } }
|
||||
zbd->stage = ZBUFFds_read;
|
||||
|
||||
case ZBUFFds_read:
|
||||
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
|
||||
if (neededInSize==0) { /* end of frame */
|
||||
zbd->stage = ZBUFFds_init;
|
||||
notDone = 0;
|
||||
break;
|
||||
}
|
||||
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
|
||||
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
|
||||
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
|
||||
ip, neededInSize);
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
ip += neededInSize;
|
||||
if (!decodedSize) break; /* this was just a header */
|
||||
zbd->outEnd = zbd->outStart + decodedSize;
|
||||
zbd->stage = ZBUFFds_flush;
|
||||
break;
|
||||
}
|
||||
if (ip==iend) { notDone = 0; break; } /* no more input */
|
||||
zbd->stage = ZBUFFds_load;
|
||||
}
|
||||
|
||||
case ZBUFFds_load:
|
||||
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
|
||||
size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
|
||||
size_t loadedSize;
|
||||
if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */
|
||||
loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip);
|
||||
ip += loadedSize;
|
||||
zbd->inPos += loadedSize;
|
||||
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
|
||||
|
||||
/* decode loaded input */
|
||||
{ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
|
||||
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
|
||||
zbd->inBuff, neededInSize);
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
zbd->inPos = 0; /* input is consumed */
|
||||
if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
|
||||
zbd->outEnd = zbd->outStart + decodedSize;
|
||||
zbd->stage = ZBUFFds_flush;
|
||||
// break; /* ZBUFFds_flush follows */
|
||||
} }
|
||||
|
||||
case ZBUFFds_flush:
|
||||
{ size_t const toFlushSize = zbd->outEnd - zbd->outStart;
|
||||
size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
|
||||
op += flushedSize;
|
||||
zbd->outStart += flushedSize;
|
||||
if (flushedSize == toFlushSize) {
|
||||
zbd->stage = ZBUFFds_read;
|
||||
if (zbd->outStart + zbd->blockSize > zbd->outBuffSize)
|
||||
zbd->outStart = zbd->outEnd = 0;
|
||||
break;
|
||||
}
|
||||
/* cannot flush everything */
|
||||
notDone = 0;
|
||||
break;
|
||||
}
|
||||
default: return ERROR(GENERIC); /* impossible */
|
||||
} }
|
||||
|
||||
/* result */
|
||||
*srcSizePtr = ip-istart;
|
||||
*dstCapacityPtr = op-ostart;
|
||||
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd);
|
||||
if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */
|
||||
nextSrcSizeHint -= zbd->inPos; /* already loaded*/
|
||||
return nextSrcSizeHint;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Tool functions
|
||||
***************************************/
|
||||
size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize /* block header size*/ ; }
|
||||
size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
|
||||
71
contrib/libzstd/include/zstd/zbuff_static.h
Normal file
71
contrib/libzstd/include/zstd/zbuff_static.h
Normal file
@ -0,0 +1,71 @@
|
||||
/*
|
||||
zstd - buffered version of compression library
|
||||
experimental complementary API, for static linking only
|
||||
Copyright (C) 2015-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net
|
||||
*/
|
||||
#ifndef ZSTD_BUFFERED_STATIC_H
|
||||
#define ZSTD_BUFFERED_STATIC_H
|
||||
|
||||
/* The objects defined into this file should be considered experimental.
|
||||
* They are not labelled stable, as their prototype may change in the future.
|
||||
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include "zstd_static.h" /* ZSTD_parameters */
|
||||
#include "zbuff.h"
|
||||
#include "zstd_internal.h" /* MIN */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Advanced Streaming functions
|
||||
***************************************/
|
||||
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize);
|
||||
|
||||
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t length = MIN(dstCapacity, srcSize);
|
||||
memcpy(dst, src, length);
|
||||
return length;
|
||||
}
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_BUFFERED_STATIC_H */
|
||||
@ -59,9 +59,15 @@ extern "C" {
|
||||
/* *************************************
|
||||
* Version
|
||||
***************************************/
|
||||
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
|
||||
#define ZSTD_VERSION_MINOR 6 /* for new (non-breaking) interface capabilities */
|
||||
#define ZSTD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
|
||||
#define ZSTD_VERSION_MAJOR 0
|
||||
#define ZSTD_VERSION_MINOR 6
|
||||
#define ZSTD_VERSION_RELEASE 1
|
||||
|
||||
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
|
||||
#define ZSTD_QUOTE(str) #str
|
||||
#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
|
||||
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
|
||||
|
||||
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
||||
ZSTDLIB_API unsigned ZSTD_versionNumber (void);
|
||||
|
||||
|
||||
72
contrib/libzstd/include/zstd/zstd_common.c
Normal file
72
contrib/libzstd/include/zstd/zstd_common.c
Normal file
@ -0,0 +1,72 @@
|
||||
/*
|
||||
Common functions of Zstd compression library
|
||||
Copyright (C) 2015-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net/
|
||||
*/
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include "error_private.h"
|
||||
#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName */
|
||||
#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Version
|
||||
******************************************/
|
||||
unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; }
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* ZSTD Error Management
|
||||
******************************************/
|
||||
/*! ZSTD_isError() :
|
||||
* tells if a return value is an error code */
|
||||
unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
/*! ZSTD_getErrorName() :
|
||||
* provides error code string from function result (useful for debugging) */
|
||||
const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
/*! ZSTD_getError() :
|
||||
* convert a `size_t` function result into a proper ZSTD_errorCode enum */
|
||||
ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
|
||||
|
||||
/*! ZSTD_getErrorString() :
|
||||
* provides error code string from enum */
|
||||
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* ZBUFF Error Management
|
||||
****************************************************************/
|
||||
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||
|
||||
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||
@ -55,7 +55,7 @@
|
||||
#include <string.h> /* memset */
|
||||
#include "mem.h"
|
||||
#include "fse_static.h"
|
||||
#include "huff0_static.h"
|
||||
#include "huf_static.h"
|
||||
#include "zstd_internal.h"
|
||||
|
||||
|
||||
@ -218,7 +218,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
const size_t maxNbSeq = blockSize / divider;
|
||||
const size_t tokenSpace = blockSize + 11*maxNbSeq;
|
||||
const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
|
||||
const size_t hSize = 1 << params.cParams.hashLog;
|
||||
const size_t hSize = ((size_t)1) << params.cParams.hashLog;
|
||||
const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
|
||||
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
||||
|
||||
@ -291,7 +291,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
|
||||
|
||||
/* copy tables */
|
||||
{ const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
|
||||
const size_t hSize = 1 << srcCCtx->params.cParams.hashLog;
|
||||
const size_t hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog;
|
||||
const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0;
|
||||
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
||||
memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace);
|
||||
@ -852,7 +852,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
|
||||
static const BYTE* g_start = NULL;
|
||||
const U32 pos = (U32)(literals - g_start);
|
||||
if (g_start==NULL) g_start = literals;
|
||||
if ((pos > 5810300) && (pos < 5810500))
|
||||
if ((pos > 2587900) && (pos < 2588050))
|
||||
printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
|
||||
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
|
||||
#endif
|
||||
@ -1619,9 +1619,6 @@ FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
|
||||
}
|
||||
}
|
||||
|
||||
/* The optimal parser */
|
||||
#include "zstd_opt.h"
|
||||
|
||||
|
||||
/* *******************************
|
||||
* Common parser - lazy strategy
|
||||
@ -1756,12 +1753,6 @@ _storeSequence:
|
||||
}
|
||||
|
||||
|
||||
|
||||
static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_compressBlock_opt_generic(ctx, src, srcSize);
|
||||
}
|
||||
|
||||
static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2);
|
||||
@ -1975,6 +1966,16 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src,
|
||||
ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* The optimal parser */
|
||||
#include "zstd_opt.h"
|
||||
|
||||
static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_compressBlock_opt_generic(ctx, src, srcSize);
|
||||
}
|
||||
|
||||
static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize);
|
||||
@ -2309,17 +2310,6 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dict
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, targetSrcSize, dictSize);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
ZSTD_adjustCParams(¶ms.cParams, targetSrcSize, dictSize);
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_targetSrcSize compressionLevel=%d\n", zc->base, compressionLevel);
|
||||
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, targetSrcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
|
||||
{
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin compressionLevel=%d\n", zc->base, compressionLevel);
|
||||
@ -2442,6 +2432,7 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
|
||||
|
||||
/*-===== Pre-defined compression levels =====-*/
|
||||
|
||||
#define ZSTD_DEFAULT_CLEVEL 5
|
||||
#define ZSTD_MAX_CLEVEL 22
|
||||
unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
|
||||
|
||||
@ -2561,7 +2552,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si
|
||||
size_t const addedSize = srcSize ? 0 : 500;
|
||||
U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1;
|
||||
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
|
||||
if (compressionLevel<=0) compressionLevel = 1;
|
||||
if (compressionLevel < 0) compressionLevel = ZSTD_DEFAULT_CLEVEL;
|
||||
if (compressionLevel==0) compressionLevel = 1;
|
||||
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
|
||||
cp = ZSTD_defaultCParameters[tableID][compressionLevel];
|
||||
if (MEM_32bits()) { /* auto-correction, for 32-bits mode */
|
||||
|
||||
@ -59,7 +59,7 @@
|
||||
#include "mem.h" /* low level memory routines */
|
||||
#include "zstd_internal.h"
|
||||
#include "fse_static.h"
|
||||
#include "huff0_static.h"
|
||||
#include "huf_static.h"
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
# include "zstd_legacy.h"
|
||||
@ -83,30 +83,20 @@
|
||||
#endif
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Macros
|
||||
***************************************/
|
||||
#define ZSTD_isError ERR_isError /* for inlining */
|
||||
#define FSE_isError ERR_isError
|
||||
#define HUF_isError ERR_isError
|
||||
|
||||
|
||||
/*_*******************************************************
|
||||
* Memory operations
|
||||
**********************************************************/
|
||||
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Error Management
|
||||
***************************************/
|
||||
unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; }
|
||||
|
||||
/*! ZSTD_isError() :
|
||||
* tells if a return value is an error code */
|
||||
unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
/*! ZSTD_getError() :
|
||||
* convert a `size_t` function result into a proper ZSTD_errorCode enum */
|
||||
ZSTD_ErrorCode ZSTD_getError(size_t code) { return ERR_getError(code); }
|
||||
|
||||
/*! ZSTD_getErrorName() :
|
||||
* provides error code string (useful for debugging) */
|
||||
const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/*-*************************************************************
|
||||
* Context management
|
||||
***************************************************************/
|
||||
@ -278,10 +268,9 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
|
||||
* @return : size of the Frame Header */
|
||||
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
|
||||
{
|
||||
U32 fcsId;
|
||||
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
||||
fcsId = (((const BYTE*)src)[4]) >> 6;
|
||||
return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId];
|
||||
{ U32 const fcsId = (((const BYTE*)src)[4]) >> 6;
|
||||
return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; }
|
||||
}
|
||||
|
||||
|
||||
@ -670,7 +659,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
|
||||
FORCE_INLINE
|
||||
size_t ZSTD_execSequence(BYTE* op,
|
||||
BYTE* const oend, seq_t sequence,
|
||||
const BYTE** litPtr, const BYTE* const litLimit_8,
|
||||
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
||||
@ -855,28 +845,22 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTD_decompress_continueDCtx() :
|
||||
* `dctx` must have been properly initialized */
|
||||
/*! ZSTD_decompressFrame() :
|
||||
* `dctx` must be properly initialized */
|
||||
static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* iend = ip + srcSize;
|
||||
const BYTE* const iend = ip + srcSize;
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + dstCapacity;
|
||||
size_t remainingSize = srcSize;
|
||||
blockProperties_t blockProperties;
|
||||
blockProperties_t blockProperties = { bt_compressed, 0 };
|
||||
|
||||
/* check */
|
||||
if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
{ const U32 magicNumber = MEM_readLE32(src);
|
||||
if (ZSTD_isLegacy(magicNumber))
|
||||
return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, magicNumber);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Frame Header */
|
||||
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
|
||||
@ -941,6 +925,12 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize)
|
||||
{
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
{ const U32 magicNumber = MEM_readLE32(src);
|
||||
if (ZSTD_isLegacy(magicNumber))
|
||||
return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize, magicNumber);
|
||||
}
|
||||
#endif
|
||||
ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
|
||||
ZSTD_checkContinuity(dctx, dst);
|
||||
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
|
||||
@ -962,7 +952,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
|
||||
regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
return regenSize;
|
||||
#else
|
||||
#else /* stack mode */
|
||||
ZSTD_DCtx dctx;
|
||||
return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);
|
||||
#endif
|
||||
@ -977,17 +967,16 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
|
||||
return dctx->expected;
|
||||
}
|
||||
|
||||
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
/* Sanity check */
|
||||
if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
|
||||
ZSTD_checkContinuity(dctx, dst);
|
||||
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
|
||||
|
||||
/* Decompress : frame header; part 1 */
|
||||
switch (dctx->stage)
|
||||
{
|
||||
case ZSTDds_getFrameHeaderSize :
|
||||
{
|
||||
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
||||
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
|
||||
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
|
||||
@ -998,10 +987,9 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
|
||||
return 0;
|
||||
}
|
||||
dctx->expected = 0; /* not necessary to copy more */
|
||||
}
|
||||
|
||||
case ZSTDds_decodeFrameHeader:
|
||||
{
|
||||
size_t result;
|
||||
{ size_t result;
|
||||
memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
|
||||
result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
|
||||
if (ZSTD_isError(result)) return result;
|
||||
@ -1010,8 +998,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
|
||||
return 0;
|
||||
}
|
||||
case ZSTDds_decodeBlockHeader:
|
||||
{
|
||||
blockProperties_t bp;
|
||||
{ blockProperties_t bp;
|
||||
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
|
||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||
if (bp.blockType == bt_end) {
|
||||
@ -1025,16 +1012,14 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
|
||||
return 0;
|
||||
}
|
||||
case ZSTDds_decompressBlock:
|
||||
{
|
||||
/* Decompress : block content */
|
||||
size_t rSize;
|
||||
{ size_t rSize;
|
||||
switch(dctx->bType)
|
||||
{
|
||||
case bt_compressed:
|
||||
rSize = ZSTD_decompressBlock_internal(dctx, dst, maxDstSize, src, srcSize);
|
||||
rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
|
||||
break;
|
||||
case bt_raw :
|
||||
rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize);
|
||||
rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
|
||||
break;
|
||||
case bt_rle :
|
||||
return ERROR(GENERIC); /* not yet handled */
|
||||
@ -1066,37 +1051,40 @@ static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
|
||||
|
||||
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
{
|
||||
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
|
||||
short offcodeNCount[MaxOff+1];
|
||||
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
|
||||
short matchlengthNCount[MaxML+1];
|
||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
||||
short litlengthNCount[MaxLL+1];
|
||||
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
||||
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize;
|
||||
|
||||
hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
|
||||
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + hSize;
|
||||
dictSize -= hSize;
|
||||
|
||||
{ short offcodeNCount[MaxOff+1];
|
||||
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
|
||||
offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
|
||||
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dict = (const char*)dict + offcodeHeaderSize;
|
||||
dictSize -= offcodeHeaderSize;
|
||||
}
|
||||
|
||||
{ short matchlengthNCount[MaxML+1];
|
||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
||||
matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
|
||||
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dict = (const char*)dict + matchlengthHeaderSize;
|
||||
dictSize -= matchlengthHeaderSize;
|
||||
}
|
||||
|
||||
{ short litlengthNCount[MaxLL+1];
|
||||
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
||||
litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
|
||||
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
}
|
||||
|
||||
dctx->flagRepeatTable = 1;
|
||||
return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
|
||||
|
||||
@ -203,10 +203,10 @@ typedef struct {
|
||||
#include ".debug/zstd_stats.h"
|
||||
#else
|
||||
typedef struct { U32 unused; } ZSTD_stats_t;
|
||||
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; };
|
||||
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; };
|
||||
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; };
|
||||
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; };
|
||||
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
|
||||
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
|
||||
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
|
||||
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
@ -242,12 +242,14 @@ typedef struct {
|
||||
U32 log2litSum;
|
||||
U32 log2offCodeSum;
|
||||
U32 factor;
|
||||
U32 cachedPrice;
|
||||
U32 cachedLitLength;
|
||||
const BYTE* cachedLiterals;
|
||||
ZSTD_stats_t stats;
|
||||
} seqStore_t;
|
||||
|
||||
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
|
||||
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
|
||||
size_t ZSTD_compressBegin_targetSrcSize(ZSTD_CCtx* zc, const void* dict, size_t dictSize, size_t targetSrcSize, int compressionLevel);
|
||||
|
||||
|
||||
#endif /* ZSTD_CCOMMON_H_MODULE */
|
||||
|
||||
@ -53,6 +53,9 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr)
|
||||
{
|
||||
unsigned u;
|
||||
|
||||
ssPtr->cachedLiterals = NULL;
|
||||
ssPtr->cachedPrice = ssPtr->cachedLitLength = 0;
|
||||
|
||||
if (ssPtr->litLengthSum == 0) {
|
||||
ssPtr->litSum = (2<<Litbits);
|
||||
ssPtr->litLengthSum = MaxLL+1;
|
||||
@ -98,17 +101,33 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr)
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals)
|
||||
FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BYTE* literals)
|
||||
{
|
||||
U32 price, u;
|
||||
|
||||
if (litLength == 0)
|
||||
return seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[0]+1);
|
||||
return ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[0]+1);
|
||||
|
||||
/* literals */
|
||||
price = litLength * seqStorePtr->log2litSum;
|
||||
if (ssPtr->cachedLiterals == literals) {
|
||||
U32 additional = litLength - ssPtr->cachedLitLength;
|
||||
const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength;
|
||||
price = ssPtr->cachedPrice + additional * ssPtr->log2litSum;
|
||||
for (u=0; u < additional; u++)
|
||||
price -= ZSTD_highbit(ssPtr->litFreq[literals2[u]]+1);
|
||||
ssPtr->cachedPrice = price;
|
||||
ssPtr->cachedLitLength = litLength;
|
||||
} else {
|
||||
price = litLength * ssPtr->log2litSum;
|
||||
for (u=0; u < litLength; u++)
|
||||
price -= ZSTD_highbit(seqStorePtr->litFreq[literals[u]]+1);
|
||||
price -= ZSTD_highbit(ssPtr->litFreq[literals[u]]+1);
|
||||
|
||||
if (litLength >= 12) {
|
||||
ssPtr->cachedLiterals = literals;
|
||||
ssPtr->cachedPrice = price;
|
||||
ssPtr->cachedLitLength = litLength;
|
||||
}
|
||||
}
|
||||
|
||||
/* literal Length */
|
||||
{ static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
|
||||
@ -121,7 +140,7 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co
|
||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||
const BYTE LL_deltaCode = 19;
|
||||
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||
price += LL_bits[llCode] + seqStorePtr->log2litLengthSum - ZSTD_highbit(seqStorePtr->litLengthFreq[llCode]+1);
|
||||
price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[llCode]+1);
|
||||
}
|
||||
|
||||
return price;
|
||||
@ -177,9 +196,10 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
|
||||
}
|
||||
|
||||
/* match offset */
|
||||
{ BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
|
||||
seqStorePtr->offCodeSum++;
|
||||
BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
|
||||
seqStorePtr->offCodeFreq[offCode]++;
|
||||
}
|
||||
|
||||
/* match Length */
|
||||
{ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
@ -241,7 +261,7 @@ static U32 ZSTD_insertBtAndGetAllMatches (
|
||||
ZSTD_CCtx* zc,
|
||||
const BYTE* const ip, const BYTE* const iLimit,
|
||||
U32 nbCompares, const U32 mls,
|
||||
U32 extDict, ZSTD_match_t* matches)
|
||||
U32 extDict, ZSTD_match_t* matches, const U32 minMatchLen)
|
||||
{
|
||||
const BYTE* const base = zc->base;
|
||||
const U32 current = (U32)(ip-base);
|
||||
@ -266,7 +286,7 @@ static U32 ZSTD_insertBtAndGetAllMatches (
|
||||
U32 mnum = 0;
|
||||
|
||||
const U32 minMatch = (mls == 3) ? 3 : 4;
|
||||
size_t bestLength = minMatch-1;
|
||||
size_t bestLength = minMatchLen-1;
|
||||
|
||||
if (minMatch == 3) { /* HC3 match finder */
|
||||
U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip);
|
||||
@ -366,26 +386,26 @@ update:
|
||||
static U32 ZSTD_BtGetAllMatches (
|
||||
ZSTD_CCtx* zc,
|
||||
const BYTE* const ip, const BYTE* const iLimit,
|
||||
const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches)
|
||||
const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen)
|
||||
{
|
||||
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */
|
||||
ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
|
||||
return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches);
|
||||
return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen);
|
||||
}
|
||||
|
||||
|
||||
static U32 ZSTD_BtGetAllMatches_selectMLS (
|
||||
ZSTD_CCtx* zc, /* Index table will be updated */
|
||||
const BYTE* ip, const BYTE* const iHighLimit,
|
||||
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches)
|
||||
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen)
|
||||
{
|
||||
switch(matchLengthSearch)
|
||||
{
|
||||
case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches);
|
||||
case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
|
||||
default :
|
||||
case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches);
|
||||
case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches);
|
||||
case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches);
|
||||
case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
|
||||
case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen);
|
||||
case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
|
||||
}
|
||||
}
|
||||
|
||||
@ -393,26 +413,26 @@ static U32 ZSTD_BtGetAllMatches_selectMLS (
|
||||
static U32 ZSTD_BtGetAllMatches_extDict (
|
||||
ZSTD_CCtx* zc,
|
||||
const BYTE* const ip, const BYTE* const iLimit,
|
||||
const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches)
|
||||
const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen)
|
||||
{
|
||||
if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */
|
||||
ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls);
|
||||
return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches);
|
||||
return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen);
|
||||
}
|
||||
|
||||
|
||||
static U32 ZSTD_BtGetAllMatches_selectMLS_extDict (
|
||||
ZSTD_CCtx* zc, /* Index table will be updated */
|
||||
const BYTE* ip, const BYTE* const iHighLimit,
|
||||
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches)
|
||||
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen)
|
||||
{
|
||||
switch(matchLengthSearch)
|
||||
{
|
||||
case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches);
|
||||
case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen);
|
||||
default :
|
||||
case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches);
|
||||
case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches);
|
||||
case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches);
|
||||
case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen);
|
||||
case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen);
|
||||
case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen);
|
||||
}
|
||||
}
|
||||
|
||||
@ -428,7 +448,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* anchor = istart;
|
||||
const BYTE* litstart;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
const BYTE* const base = ctx->base;
|
||||
@ -442,10 +461,9 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
ZSTD_optimal_t* opt = seqStorePtr->priceTable;
|
||||
ZSTD_match_t* matches = seqStorePtr->matchTable;
|
||||
const BYTE* inr;
|
||||
U32 cur, match_num, last_pos, litlen, price;
|
||||
|
||||
/* init */
|
||||
U32 rep[ZSTD_REP_INIT];
|
||||
U32 offset, rep[ZSTD_REP_INIT];
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; }
|
||||
|
||||
ctx->nextToUpdate3 = ctx->nextToUpdate;
|
||||
@ -457,47 +475,37 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
|
||||
/* Match Loop */
|
||||
while (ip < ilimit) {
|
||||
U32 u;
|
||||
U32 mlen=0;
|
||||
U32 best_mlen=0;
|
||||
U32 best_off=0;
|
||||
U32 cur, match_num, last_pos, litlen, price;
|
||||
U32 u, mlen, best_mlen, best_off, litLength;
|
||||
memset(opt, 0, sizeof(ZSTD_optimal_t));
|
||||
last_pos = 0;
|
||||
inr = ip;
|
||||
litstart = ((U32)(ip - anchor) > 128) ? ip - 128 : anchor;
|
||||
opt[0].litlen = (U32)(ip - litstart);
|
||||
litlen = (U32)(ip - anchor);
|
||||
|
||||
/* check repCode */
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++)
|
||||
if (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - rep[i], minMatch)) {
|
||||
/* repcode : we take it */
|
||||
mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-rep[i], iend) + minMatch;
|
||||
|
||||
ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen);
|
||||
if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
|
||||
best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
|
||||
goto _storeSequence;
|
||||
}
|
||||
|
||||
best_off = (i<=1 && ip == anchor) ? 1-i : i;
|
||||
litlen = opt[0].litlen;
|
||||
do {
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - MINMATCH);
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
|
||||
if (mlen > last_pos || price < opt[mlen].price)
|
||||
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
|
||||
mlen--;
|
||||
} while (mlen >= minMatch);
|
||||
} }
|
||||
|
||||
match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */
|
||||
match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch);
|
||||
|
||||
ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos);
|
||||
if (!last_pos && !match_num) { ip++; continue; }
|
||||
|
||||
{ U32 i ; for (i=0; i<ZSTD_REP_INIT; i++) opt[0].rep[i] = rep[i]; }
|
||||
opt[0].mlen = 1;
|
||||
|
||||
if (match_num && matches[match_num-1].len > sufficient_len) {
|
||||
if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) {
|
||||
best_mlen = matches[match_num-1].len;
|
||||
best_off = matches[match_num-1].off;
|
||||
cur = 0;
|
||||
@ -505,23 +513,26 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
goto _storeSequence;
|
||||
}
|
||||
|
||||
/* set prices using matches at position = 0 */
|
||||
best_mlen = (last_pos) ? last_pos : minMatch;
|
||||
|
||||
// set prices using matches at position = 0
|
||||
for (u = 0; u < match_num; u++) {
|
||||
mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
|
||||
best_mlen = (matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM;
|
||||
best_mlen = matches[u].len;
|
||||
ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos);
|
||||
litlen = opt[0].litlen;
|
||||
while (mlen <= best_mlen) {
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - MINMATCH);
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
|
||||
if (mlen > last_pos || price < opt[mlen].price)
|
||||
SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
|
||||
SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */
|
||||
mlen++;
|
||||
} }
|
||||
|
||||
if (last_pos < minMatch) { ip++; continue; }
|
||||
|
||||
/* initialize opt[0] */
|
||||
{ U32 i ; for (i=0; i<ZSTD_REP_INIT; i++) opt[0].rep[i] = rep[i]; }
|
||||
opt[0].mlen = 1;
|
||||
opt[0].litlen = litlen;
|
||||
|
||||
/* check further positions */
|
||||
for (cur = 1; cur <= last_pos; cur++) {
|
||||
inr = ip + cur;
|
||||
@ -531,7 +542,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
if (cur > litlen) {
|
||||
price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen);
|
||||
} else
|
||||
price = ZSTD_getLiteralPrice(seqStorePtr, litlen, litstart);
|
||||
price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
|
||||
} else {
|
||||
litlen = 1;
|
||||
price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1);
|
||||
@ -559,8 +570,8 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
}
|
||||
|
||||
ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
|
||||
best_mlen = 0;
|
||||
|
||||
best_mlen = minMatch;
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++)
|
||||
if (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - opt[cur].rep[i], minMatch)) { /* check rep */
|
||||
mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - opt[cur].rep[i], iend) + minMatch;
|
||||
@ -578,13 +589,13 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
if (cur > litlen) {
|
||||
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
|
||||
} else
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - MINMATCH);
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
|
||||
} else {
|
||||
litlen = 0;
|
||||
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
|
||||
}
|
||||
|
||||
best_mlen = mlen;
|
||||
if (mlen > best_mlen) best_mlen = mlen;
|
||||
ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen);
|
||||
|
||||
do {
|
||||
@ -594,23 +605,20 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
} while (mlen >= minMatch);
|
||||
} }
|
||||
|
||||
|
||||
match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches);
|
||||
match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen);
|
||||
ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num);
|
||||
|
||||
if (match_num > 0 && matches[match_num-1].len > sufficient_len) {
|
||||
if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) {
|
||||
best_mlen = matches[match_num-1].len;
|
||||
best_off = matches[match_num-1].off;
|
||||
last_pos = cur + 1;
|
||||
goto _storeSequence;
|
||||
}
|
||||
|
||||
best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch;
|
||||
|
||||
/* set prices using matches at position = cur */
|
||||
for (u = 0; u < match_num; u++) {
|
||||
mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
|
||||
best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur;
|
||||
best_mlen = matches[u].len;
|
||||
|
||||
// ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos);
|
||||
while (mlen <= best_mlen) {
|
||||
@ -619,7 +627,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
if (cur > litlen)
|
||||
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH);
|
||||
else
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - MINMATCH);
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
|
||||
} else {
|
||||
litlen = 0;
|
||||
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH);
|
||||
@ -643,7 +651,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]);
|
||||
|
||||
opt[0].mlen = 1;
|
||||
U32 offset;
|
||||
|
||||
while (1) {
|
||||
mlen = opt[cur].mlen;
|
||||
@ -667,8 +674,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
if (mlen == 1) { ip++; cur++; continue; }
|
||||
offset = opt[cur].off;
|
||||
cur += mlen;
|
||||
|
||||
U32 const litLength = (U32)(ip - anchor);
|
||||
litLength = (U32)(ip - anchor);
|
||||
// ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
|
||||
|
||||
if (offset >= ZSTD_REP_NUM) {
|
||||
@ -723,7 +729,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* anchor = istart;
|
||||
const BYTE* litstart;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
const BYTE* const base = ctx->base;
|
||||
@ -740,10 +745,9 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
ZSTD_optimal_t* opt = seqStorePtr->priceTable;
|
||||
ZSTD_match_t* matches = seqStorePtr->matchTable;
|
||||
const BYTE* inr;
|
||||
U32 cur, match_num, last_pos, litlen, price;
|
||||
|
||||
/* init */
|
||||
U32 rep[ZSTD_REP_INIT];
|
||||
U32 offset, rep[ZSTD_REP_INIT];
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; }
|
||||
|
||||
ctx->nextToUpdate3 = ctx->nextToUpdate;
|
||||
@ -755,16 +759,13 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
|
||||
/* Match Loop */
|
||||
while (ip < ilimit) {
|
||||
U32 u;
|
||||
U32 mlen=0;
|
||||
U32 best_mlen=0;
|
||||
U32 best_off=0;
|
||||
U32 cur, match_num, last_pos, litlen, price;
|
||||
U32 u, mlen, best_mlen, best_off, litLength;
|
||||
U32 current = (U32)(ip-base);
|
||||
memset(opt, 0, sizeof(ZSTD_optimal_t));
|
||||
last_pos = 0;
|
||||
inr = ip;
|
||||
litstart = ((U32)(ip - anchor) > 128) ? ip - 128 : anchor;
|
||||
opt[0].litlen = (U32)(ip - litstart);
|
||||
opt[0].litlen = (U32)(ip - anchor);
|
||||
|
||||
/* check repCode */
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) {
|
||||
@ -786,14 +787,14 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
best_off = (i<=1 && ip == anchor) ? 1-i : i;
|
||||
litlen = opt[0].litlen;
|
||||
do {
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - MINMATCH);
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
|
||||
if (mlen > last_pos || price < opt[mlen].price)
|
||||
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
|
||||
mlen--;
|
||||
} while (mlen >= minMatch);
|
||||
} } }
|
||||
|
||||
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */
|
||||
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */
|
||||
|
||||
ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos);
|
||||
if (!last_pos && !match_num) { ip++; continue; }
|
||||
@ -801,7 +802,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) opt[0].rep[i] = rep[i]; }
|
||||
opt[0].mlen = 1;
|
||||
|
||||
if (match_num && matches[match_num-1].len > sufficient_len) {
|
||||
if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) {
|
||||
best_mlen = matches[match_num-1].len;
|
||||
best_off = matches[match_num-1].off;
|
||||
cur = 0;
|
||||
@ -814,11 +815,11 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
// set prices using matches at position = 0
|
||||
for (u = 0; u < match_num; u++) {
|
||||
mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
|
||||
best_mlen = (matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM;
|
||||
best_mlen = matches[u].len;
|
||||
ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos);
|
||||
litlen = opt[0].litlen;
|
||||
while (mlen <= best_mlen) {
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - MINMATCH);
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
|
||||
if (mlen > last_pos || price < opt[mlen].price)
|
||||
SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
|
||||
mlen++;
|
||||
@ -838,7 +839,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
if (cur > litlen) {
|
||||
price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen);
|
||||
} else
|
||||
price = ZSTD_getLiteralPrice(seqStorePtr, litlen, litstart);
|
||||
price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor);
|
||||
} else {
|
||||
litlen = 1;
|
||||
price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1);
|
||||
@ -891,7 +892,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
if (cur > litlen) {
|
||||
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
|
||||
} else
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, litstart, best_off, mlen - MINMATCH);
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
|
||||
} else {
|
||||
litlen = 0;
|
||||
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
|
||||
@ -907,7 +908,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
} while (mlen >= minMatch);
|
||||
} } }
|
||||
|
||||
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches);
|
||||
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
|
||||
ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num);
|
||||
|
||||
if (match_num > 0 && matches[match_num-1].len > sufficient_len) {
|
||||
@ -931,7 +932,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
if (cur > litlen)
|
||||
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH);
|
||||
else
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, litstart, matches[u].off, mlen - MINMATCH);
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
|
||||
} else {
|
||||
litlen = 0;
|
||||
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH);
|
||||
@ -955,7 +956,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]);
|
||||
|
||||
opt[0].mlen = 1;
|
||||
U32 offset;
|
||||
|
||||
while (1) {
|
||||
mlen = opt[cur].mlen;
|
||||
@ -979,8 +979,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
if (mlen == 1) { ip++; cur++; continue; }
|
||||
offset = opt[cur].off;
|
||||
cur += mlen;
|
||||
|
||||
U32 const litLength = (U32)(ip - anchor);
|
||||
litLength = (U32)(ip - anchor);
|
||||
// ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
|
||||
|
||||
if (offset >= ZSTD_REP_NUM) {
|
||||
|
||||
@ -103,7 +103,7 @@ ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
|
||||
/*! ZSTD_getCParams() :
|
||||
* @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
|
||||
* `srcSize` value is optional, select 0 if not known */
|
||||
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize);
|
||||
ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize);
|
||||
|
||||
/*! ZSTD_checkParams() :
|
||||
* Ensure param values remain within authorized range */
|
||||
@ -250,8 +250,8 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
|
||||
*/
|
||||
|
||||
#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
|
||||
size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/*-*************************************
|
||||
@ -261,7 +261,8 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, cons
|
||||
/*! ZSTD_getErrorCode() :
|
||||
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
|
||||
which can be used to compare directly with enum list published into "error_public.h" */
|
||||
ZSTD_ErrorCode ZSTD_getError(size_t code);
|
||||
ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
|
||||
ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user