mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 15:42:02 +00:00
Updated zstd library to v0.1.1 [#METR-2807].
This commit is contained in:
parent
217ef544c2
commit
34a5cf045f
@ -1 +1,5 @@
|
||||
https://github.com/Cyan4973/zstd/tree/1eca5f52994434d3b0427c9014403cf01495f54a/
|
||||
https://github.com/Cyan4973/zstd/tree/v0.1.1
|
||||
|
||||
Added
|
||||
#pragma GCC diagnostic ignored "-Warray-bounds"
|
||||
as noted here: http://fastcompression.blogspot.ru/2015/08/fuzz-testing-zstandard.html?showComment=1441763002674#c8204923539141394992
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -55,12 +55,11 @@ size_t FSE_decompress(void* dst, size_t maxDstSize,
|
||||
/*
|
||||
FSE_compress():
|
||||
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
|
||||
'dst' buffer must be already allocated, and sized to handle worst case situations.
|
||||
Worst case size evaluation is provided by FSE_compressBound().
|
||||
return : size of compressed data
|
||||
Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
|
||||
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
|
||||
if FSE_isError(return), it's an error code.
|
||||
'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= FSE_compressBound(srcSize)
|
||||
return : size of compressed data (<= maxDstSize)
|
||||
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
||||
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
|
||||
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
|
||||
|
||||
FSE_decompress():
|
||||
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
|
||||
@ -70,7 +69,33 @@ FSE_decompress():
|
||||
|
||||
** Important ** : FSE_decompress() doesn't decompress non-compressible nor RLE data !!!
|
||||
Why ? : making this distinction requires a header.
|
||||
FSE library doesn't manage headers, which are intentionally left to the user layer.
|
||||
Header management is intentionally delegated to the user layer, which can better manage special cases.
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
* Huff0 simple functions
|
||||
******************************************/
|
||||
size_t HUF_compress(void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize);
|
||||
size_t HUF_decompress(void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
/*
|
||||
HUF_compress():
|
||||
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
|
||||
'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= HUF_compressBound(srcSize)
|
||||
return : size of compressed data (<= maxDstSize)
|
||||
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
||||
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
|
||||
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
|
||||
|
||||
HUF_decompress():
|
||||
Decompress Huff0 data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated destination buffer 'dst', of size 'maxDstSize'.
|
||||
return : size of regenerated data (<= maxDstSize)
|
||||
or an error code, which can be tested using FSE_isError()
|
||||
|
||||
** Important ** : HUF_decompress() doesn't decompress non-compressible nor RLE data !!!
|
||||
*/
|
||||
|
||||
|
||||
@ -98,6 +123,8 @@ FSE_compress2():
|
||||
*/
|
||||
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
|
||||
/******************************************
|
||||
* FSE detailed API
|
||||
@ -106,18 +133,18 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
|
||||
FSE_compress() does the following:
|
||||
1. count symbol occurrence from source[] into table count[]
|
||||
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
|
||||
3. save normalized counters to memory buffer using writeHeader()
|
||||
3. save normalized counters to memory buffer using writeNCount()
|
||||
4. build encoding table 'CTable' from normalized counters
|
||||
5. encode the data stream using encoding table 'CTable'
|
||||
|
||||
FSE_decompress() does the following:
|
||||
1. read normalized counters with readHeader()
|
||||
1. read normalized counters with readNCount()
|
||||
2. build decoding table 'DTable' from normalized counters
|
||||
3. decode the data stream using decoding table 'DTable'
|
||||
|
||||
The following API allows to trigger specific sub-functions for advanced tasks.
|
||||
The following API allows targeting specific sub-functions for advanced tasks.
|
||||
For example, it's possible to compress several blocks using the same 'CTable',
|
||||
or to save and provide normalized distribution using one's own method.
|
||||
or to save and provide normalized distribution using external method.
|
||||
*/
|
||||
|
||||
/* *** COMPRESSION *** */
|
||||
@ -163,8 +190,8 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized
|
||||
|
||||
/*
|
||||
Constructor and Destructor of type FSE_CTable
|
||||
Not that its size depends on parameters 'tableLog' and 'maxSymbolValue' */
|
||||
typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void */
|
||||
Note that its size depends on 'tableLog' and 'maxSymbolValue' */
|
||||
typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void* */
|
||||
FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
|
||||
void FSE_freeCTable (FSE_CTable* ct);
|
||||
|
||||
@ -173,30 +200,32 @@ FSE_buildCTable():
|
||||
Builds 'ct', which must be already allocated, using FSE_createCTable()
|
||||
return : 0
|
||||
or an errorCode, which can be tested using FSE_isError() */
|
||||
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*
|
||||
FSE_compress_usingCTable():
|
||||
Compress 'src' using 'ct' into 'dst' which must be already allocated
|
||||
return : size of compressed data
|
||||
return : size of compressed data (<= maxDstSize)
|
||||
or 0 if compressed data could not fit into 'dst'
|
||||
or an errorCode, which can be tested using FSE_isError() */
|
||||
size_t FSE_compress_usingCTable (void* dst, size_t dstSize, const void* src, size_t srcSize, const FSE_CTable* ct);
|
||||
size_t FSE_compress_usingCTable (void* dst, size_t maxDstSize, const void* src, size_t srcSize, const FSE_CTable* ct);
|
||||
|
||||
/*
|
||||
Tutorial :
|
||||
----------
|
||||
The first step is to count all symbols. FSE_count() provides one quick way to do this job.
|
||||
The first step is to count all symbols. FSE_count() does this job very fast.
|
||||
Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
|
||||
'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
|
||||
maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
|
||||
FSE_count() will return the number of occurrence of the most frequent symbol.
|
||||
This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
|
||||
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
|
||||
|
||||
The next step is to normalize the frequencies.
|
||||
FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
|
||||
It also guarantees a minimum of 1 to any Symbol which frequency is >= 1.
|
||||
You can use input 'tableLog'==0 to mean "use default tableLog value".
|
||||
If you are unsure of which tableLog value to use, you can optionally call FSE_optimalTableLog(),
|
||||
It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
|
||||
You can use 'tableLog'==0 to mean "use default tableLog value".
|
||||
If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
|
||||
which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
|
||||
|
||||
The result of FSE_normalizeCount() will be saved into a table,
|
||||
@ -204,23 +233,23 @@ called 'normalizedCounter', which is a table of signed short.
|
||||
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
|
||||
The return value is tableLog if everything proceeded as expected.
|
||||
It is 0 if there is a single symbol within distribution.
|
||||
If there is an error(typically, invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
|
||||
If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
|
||||
|
||||
'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeHeader().
|
||||
'header' buffer must be already allocated.
|
||||
'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
|
||||
'buffer' must be already allocated.
|
||||
For guaranteed success, buffer size must be at least FSE_headerBound().
|
||||
The result of the function is the number of bytes written into 'header'.
|
||||
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()) (for example, buffer size too small).
|
||||
The result of the function is the number of bytes written into 'buffer'.
|
||||
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
|
||||
|
||||
'normalizedCounter' can then be used to create the compression table 'CTable'.
|
||||
The space required by 'CTable' must be already allocated. Its size is provided by FSE_sizeof_CTable().
|
||||
'CTable' must be aligned of 4 bytes boundaries.
|
||||
The space required by 'CTable' must be already allocated, using FSE_createCTable().
|
||||
You can then use FSE_buildCTable() to fill 'CTable'.
|
||||
In both cases, if there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
|
||||
If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
|
||||
|
||||
'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
|
||||
Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
|
||||
The function returns the size of compressed data (without header).
|
||||
The function returns the size of compressed data (without header), necessarily <= maxDstSize.
|
||||
If it returns '0', compressed data could not fit into 'dst'.
|
||||
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
|
||||
*/
|
||||
|
||||
@ -237,26 +266,25 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, un
|
||||
|
||||
/*
|
||||
Constructor and Destructor of type FSE_DTable
|
||||
Note that its size depends on parameters 'tableLog' */
|
||||
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void */
|
||||
Note that its size depends on 'tableLog' */
|
||||
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
|
||||
FSE_DTable* FSE_createDTable(unsigned tableLog);
|
||||
void FSE_freeDTable(FSE_DTable* dt);
|
||||
|
||||
/*
|
||||
FSE_buildDTable():
|
||||
Builds 'dt', which must be already allocated, using FSE_createDTable()
|
||||
return : 1 if 'dt' is compatible with fast mode, 0 otherwise,
|
||||
return : 0,
|
||||
or an errorCode, which can be tested using FSE_isError() */
|
||||
size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*
|
||||
FSE_decompress_usingDTable():
|
||||
Decompress compressed source 'cSrc' of size 'cSrcSize'
|
||||
using 'dt' into 'dst' which must be already allocated.
|
||||
Use fastMode==1 only if authorized by result of FSE_buildDTable().
|
||||
Decompress compressed source 'cSrc' of size 'cSrcSize' using 'dt'
|
||||
into 'dst' which must be already allocated.
|
||||
return : size of regenerated data (necessarily <= maxDstSize)
|
||||
or an errorCode, which can be tested using FSE_isError() */
|
||||
size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt, size_t fastMode);
|
||||
size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
|
||||
|
||||
/*
|
||||
Tutorial :
|
||||
@ -266,26 +294,24 @@ Tutorial :
|
||||
If block is a single repeated byte, use memset() instead )
|
||||
|
||||
The first step is to obtain the normalized frequencies of symbols.
|
||||
This can be performed by reading a header with FSE_readHeader().
|
||||
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of short.
|
||||
This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
|
||||
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
|
||||
In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
|
||||
or size the table to handle worst case situations (typically 256).
|
||||
FSE_readHeader will provide 'tableLog' and 'maxSymbolValue' stored into the header.
|
||||
The result of FSE_readHeader() is the number of bytes read from 'header'.
|
||||
Note that 'headerSize' must be at least 4 bytes, even if useful information is less than that.
|
||||
FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
|
||||
The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
|
||||
Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
|
||||
If there is an error, the function will return an error code, which can be tested using FSE_isError().
|
||||
|
||||
The next step is to create the decompression tables 'FSE_DTable' from 'normalizedCounter'.
|
||||
The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
|
||||
This is performed by the function FSE_buildDTable().
|
||||
The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
|
||||
The function will return 1 if FSE_DTable is compatible with fastMode, 0 otherwise.
|
||||
If there is an error, the function will return an error code, which can be tested using FSE_isError().
|
||||
|
||||
'FSE_DTable' can then be used to decompress 'cSrc', with FSE_decompress_usingDTable().
|
||||
Only trigger fastMode if it was authorized by the result of FSE_buildDTable(), otherwise decompression will fail.
|
||||
cSrcSize must be correct, otherwise decompression will fail.
|
||||
FSE_decompress_usingDTable() result will tell how many bytes were regenerated.
|
||||
If there is an error, the function will return an error code, which can be tested using FSE_isError().
|
||||
'cSrcSize' must be strictly correct, otherwise decompression will fail.
|
||||
FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=maxDstSize).
|
||||
If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
|
||||
*/
|
||||
|
||||
|
||||
|
@ -48,12 +48,25 @@ extern "C" {
|
||||
/******************************************
|
||||
* Static allocation
|
||||
******************************************/
|
||||
#define FSE_MAX_HEADERSIZE 512
|
||||
#define FSE_COMPRESSBOUND(size) (size + (size>>7) + FSE_MAX_HEADERSIZE) /* Macro can be useful for static allocation */
|
||||
/* You can statically allocate a CTable as a table of unsigned using below macro */
|
||||
/* FSE buffer bounds */
|
||||
#define FSE_NCOUNTBOUND 512
|
||||
#define FSE_BLOCKBOUND(size) (size + (size>>7))
|
||||
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* You can statically allocate FSE CTable/DTable as a table of unsigned using below macro */
|
||||
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
|
||||
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
||||
|
||||
/* Huff0 buffer bounds */
|
||||
#define HUF_CTABLEBOUND 129
|
||||
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if pre-filtered with fast heuristic */
|
||||
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* You can statically allocate Huff0 DTable as a table of unsigned short using below macro */
|
||||
#define HUF_DTABLE_SIZE_U16(maxTableLog) (1 + (1<<maxTableLog))
|
||||
#define HUF_CREATE_STATIC_DTABLE(DTable, maxTableLog) \
|
||||
unsigned short DTable[HUF_DTABLE_SIZE_U16(maxTableLog)] = { maxTableLog }
|
||||
|
||||
|
||||
/******************************************
|
||||
* Error Management
|
||||
@ -96,6 +109,7 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
|
||||
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
|
||||
Visual seems to do it automatically.
|
||||
For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
|
||||
If none of these solutions is applicable, include "fse.c" directly.
|
||||
*/
|
||||
|
||||
typedef struct
|
||||
@ -104,6 +118,7 @@ typedef struct
|
||||
int bitPos;
|
||||
char* startPtr;
|
||||
char* ptr;
|
||||
char* endPtr;
|
||||
} FSE_CStream_t;
|
||||
|
||||
typedef struct
|
||||
@ -114,10 +129,10 @@ typedef struct
|
||||
unsigned stateLog;
|
||||
} FSE_CState_t;
|
||||
|
||||
void FSE_initCStream(FSE_CStream_t* bitC, void* dstBuffer);
|
||||
size_t FSE_initCStream(FSE_CStream_t* bitC, void* dstBuffer, size_t maxDstSize);
|
||||
void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
|
||||
|
||||
void FSE_encodeSymbol(FSE_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned char symbol);
|
||||
void FSE_encodeSymbol(FSE_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
|
||||
void FSE_addBits(FSE_CStream_t* bitC, size_t value, unsigned nbBits);
|
||||
void FSE_flushBits(FSE_CStream_t* bitC);
|
||||
|
||||
@ -133,17 +148,18 @@ So the first symbol you will encode is the last you will decode, like a LIFO sta
|
||||
|
||||
You will need a few variables to track your CStream. They are :
|
||||
|
||||
FSE_CTable ct; // Provided by FSE_buildCTable()
|
||||
FSE_CStream_t bitC; // bitStream tracking structure
|
||||
FSE_CState_t state; // State tracking structure (can have several)
|
||||
FSE_CTable ct; // Provided by FSE_buildCTable()
|
||||
FSE_CStream_t bitStream; // bitStream tracking structure
|
||||
FSE_CState_t state; // State tracking structure (can have several)
|
||||
|
||||
|
||||
The first thing to do is to init bitStream and state.
|
||||
FSE_initCStream(&bitC, dstBuffer);
|
||||
size_t errorCode = FSE_initCStream(&bitStream, dstBuffer, maxDstSize);
|
||||
FSE_initCState(&state, ct);
|
||||
|
||||
Note that FSE_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
|
||||
You can then encode your input data, byte after byte.
|
||||
FSE_encodeByte() outputs a maximum of 'tableLog' bits at a time.
|
||||
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
|
||||
Remember decoding will be done in reverse direction.
|
||||
FSE_encodeByte(&bitStream, &state, symbol);
|
||||
|
||||
@ -159,8 +175,9 @@ Writing data to memory is a manual operation, performed by the flushBits functio
|
||||
Your last FSE encoding operation shall be to flush your last state value(s).
|
||||
FSE_flushState(&bitStream, &state);
|
||||
|
||||
Finally, you must then close the bitStream.
|
||||
The function returns the size in bytes of CStream.
|
||||
Finally, you must close the bitStream.
|
||||
The function returns the size of CStream in bytes.
|
||||
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
|
||||
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
|
||||
size_t size = FSE_closeCStream(&bitStream);
|
||||
*/
|
||||
@ -194,6 +211,12 @@ unsigned int FSE_reloadDStream(FSE_DStream_t* bitD);
|
||||
unsigned FSE_endOfDStream(const FSE_DStream_t* bitD);
|
||||
unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
||||
|
||||
typedef enum { FSE_DStream_unfinished = 0,
|
||||
FSE_DStream_endOfBuffer = 1,
|
||||
FSE_DStream_completed = 2,
|
||||
FSE_DStream_tooFar = 3 } FSE_DStream_status; /* result of FSE_reloadDStream() */
|
||||
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... ?! */
|
||||
|
||||
/*
|
||||
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSE-encoded symbols from the bitStream,
|
||||
@ -201,16 +224,16 @@ and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
FSE_DStream_t DStream; // Stream context
|
||||
FSE_DState_t DState; // State context. Multiple ones are possible
|
||||
FSE_DTable dt; // Decoding table, provided by FSE_buildDTable()
|
||||
U32 tableLog; // Provided by FSE_readHeader()
|
||||
FSE_DStream_t DStream; // Stream context
|
||||
FSE_DState_t DState; // State context. Multiple ones are possible
|
||||
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = FSE_initDStream(&DStream, &optionalId, srcBuffer, srcSize);
|
||||
errorCode = FSE_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s) :
|
||||
errorCode = FSE_initDState(&DState, &DStream, dt, tableLog);
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
||||
|
||||
You can then decode your data, symbol after symbol.
|
||||
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
||||
@ -218,28 +241,28 @@ Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last
|
||||
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25
|
||||
unsigned int bitField = FSE_readBits(&DStream, nbBits);
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = FSE_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size is controlled by bitD_t==32 bits).
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSE_reloadDStream(&DStream);
|
||||
|
||||
FSE_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
0 : there is still some data left into the DStream.
|
||||
1 : Dstream reached end of buffer, but is not yet fully extracted. It will not load data from memory any more.
|
||||
2 : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
3 : Dstream went too far. Decompression result is corrupted.
|
||||
FSE_DStream_unfinished : there is still some data left into the DStream.
|
||||
FSE_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
FSE_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
FSE_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer(1), progress slowly, notably if you decode multiple symbols per loop,
|
||||
When reaching end of buffer (FSE_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
FSE_reloadDStream(&DStream) >= 2
|
||||
FSE_reloadDStream(&DStream) >= FSE_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
FSE_endOfDStream(&DStream);
|
||||
Check also the states. There might be some entropy left there, able to decode some high probability (>50%) symbol.
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSE_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
@ -251,7 +274,7 @@ size_t FSE_readBitsFast(FSE_DStream_t* bitD, unsigned nbBits);
|
||||
/* faster, but works only if nbBits >= 1 (otherwise, result will be corrupted) */
|
||||
|
||||
unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD);
|
||||
/* faster, but works only if nbBits >= 1 (otherwise, result will be corrupted) */
|
||||
/* faster, but works only if allways nbBits >= 1 (otherwise, result will be corrupted) */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -46,8 +46,8 @@ extern "C" {
|
||||
* Version
|
||||
**************************************/
|
||||
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
|
||||
#define ZSTD_VERSION_MINOR 0 /* for new (non-breaking) interface capabilities */
|
||||
#define ZSTD_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
|
||||
#define ZSTD_VERSION_MINOR 1 /* for new (non-breaking) interface capabilities */
|
||||
#define ZSTD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
|
||||
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
||||
unsigned ZSTD_versionNumber (void);
|
||||
|
||||
@ -64,8 +64,8 @@ size_t ZSTD_decompress( void* dst, size_t maxOriginalSize,
|
||||
/*
|
||||
ZSTD_compress() :
|
||||
Compresses 'srcSize' bytes from buffer 'src' into buffer 'dst', of maximum size 'dstSize'.
|
||||
Destination buffer should be sized to handle worst cases situations (input data not compressible).
|
||||
Worst case size evaluation is provided by function ZSTD_compressBound().
|
||||
Destination buffer must be already allocated.
|
||||
Compression runs faster if maxDstSize >= ZSTD_compressBound(srcSize).
|
||||
return : the number of bytes written into buffer 'dst'
|
||||
or an error code if it fails (which can be tested using ZSTD_isError())
|
||||
|
||||
|
@ -74,9 +74,9 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
|
||||
**************************************/
|
||||
#define ZSTD_LIST_ERRORS(ITEM) \
|
||||
ITEM(ZSTD_OK_NoError) ITEM(ZSTD_ERROR_GENERIC) \
|
||||
ITEM(ZSTD_ERROR_wrongMagicNumber) \
|
||||
ITEM(ZSTD_ERROR_wrongSrcSize) ITEM(ZSTD_ERROR_maxDstSize_tooSmall) \
|
||||
ITEM(ZSTD_ERROR_wrongLBlockSize) \
|
||||
ITEM(ZSTD_ERROR_MagicNumber) \
|
||||
ITEM(ZSTD_ERROR_SrcSize) ITEM(ZSTD_ERROR_maxDstSize_tooSmall) \
|
||||
ITEM(ZSTD_ERROR_corruption) \
|
||||
ITEM(ZSTD_ERROR_maxCode)
|
||||
|
||||
#define ZSTD_GENERATE_ENUM(ENUM) ENUM,
|
||||
|
Loading…
Reference in New Issue
Block a user