fse.h File Reference

#include "zstd_deps.h"

Go to the source code of this file.

Macros
#define	FSE_H
#define	FSE_PUBLIC_API
#define	FSE_VERSION_MAJOR   0
#define	FSE_VERSION_MINOR   9
#define	FSE_VERSION_RELEASE   0
#define	FSE_LIB_VERSION   FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
#define	FSE_QUOTE(str)
#define	FSE_EXPAND_AND_QUOTE(str)
#define	FSE_VERSION_STRING   FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
#define	FSE_VERSION_NUMBER   (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)

Typedefs
typedef unsigned	FSE_CTable
typedef unsigned	FSE_DTable

Functions
FSE_PUBLIC_API unsigned	FSE_versionNumber (void)
FSE_PUBLIC_API size_t	FSE_compress (void *dst, size_t dstCapacity, const void *src, size_t srcSize)
FSE_PUBLIC_API size_t	FSE_decompress (void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize)
FSE_PUBLIC_API size_t	FSE_compressBound (size_t size)
FSE_PUBLIC_API unsigned	FSE_isError (size_t code)
FSE_PUBLIC_API const char *	FSE_getErrorName (size_t code)
FSE_PUBLIC_API size_t	FSE_compress2 (void *dst, size_t dstSize, const void *src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API unsigned	FSE_optimalTableLog (unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
FSE_PUBLIC_API size_t	FSE_normalizeCount (short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount)
FSE_PUBLIC_API size_t	FSE_NCountWriteBound (unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API size_t	FSE_writeNCount (void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API FSE_CTable *	FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API void	FSE_freeCTable (FSE_CTable *ct)
FSE_PUBLIC_API size_t	FSE_buildCTable (FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API size_t	FSE_compress_usingCTable (void *dst, size_t dstCapacity, const void *src, size_t srcSize, const FSE_CTable *ct)
FSE_PUBLIC_API size_t	FSE_readNCount (short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize)
FSE_PUBLIC_API size_t	FSE_readNCount_bmi2 (short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize, int bmi2)
FSE_PUBLIC_API FSE_DTable *	FSE_createDTable (unsigned tableLog)
FSE_PUBLIC_API void	FSE_freeDTable (FSE_DTable *dt)
FSE_PUBLIC_API size_t	FSE_buildDTable (FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API size_t	FSE_decompress_usingDTable (void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt)

Macro Definition Documentation

FSE_EXPAND_AND_QUOTE

#define FSE_EXPAND_AND_QUOTE

(

str

)

Value:

FSE_QUOTE(str)

FSE_H

#define FSE_H

FSE_LIB_VERSION

#define FSE_LIB_VERSION   FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE

FSE_PUBLIC_API

#define FSE_PUBLIC_API

FSE_QUOTE

#define FSE_QUOTE

(

str

)

Value:

#str

FSE_VERSION_MAJOR

#define FSE_VERSION_MAJOR   0

FSE_VERSION_MINOR

#define FSE_VERSION_MINOR   9

FSE_VERSION_NUMBER

#define FSE_VERSION_NUMBER   (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)

FSE_VERSION_RELEASE

#define FSE_VERSION_RELEASE   0

FSE_VERSION_STRING

#define FSE_VERSION_STRING   FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)

Typedef Documentation

FSE_CTable

typedef unsigned FSE_CTable

Constructor and Destructor of FSE_CTable. Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue'

FSE_DTable

typedef unsigned FSE_DTable

Constructor and Destructor of FSE_DTable. Note that its size depends on 'tableLog'

Constructor and Destructor of type FSE_DTable Note that its size depends on 'tableLog'

Function Documentation

FSE_buildCTable()

FSE_PUBLIC_API size_t FSE_buildCTable	(	FSE_CTable *	ct,
		const short *	normalizedCounter,
		unsigned	maxSymbolValue,
		unsigned	tableLog )

FSE_buildCTable(): Builds ct, which must be already allocated, using FSE_createCTable().

Returns

: 0, or an errorCode, which can be tested using FSE_isError()

FSE_buildDTable()

static size_t FSE_buildDTable	(	FSE_DTable *	dt,
		const short *	normalizedCounter,
		unsigned	maxSymbolValue,
		unsigned	tableLog )

FSE_buildDTable(): Builds 'dt', which must be already allocated, using FSE_createDTable(). return : 0, or an errorCode, which can be tested using FSE_isError()

FSE_compress()

FSE_PUBLIC_API size_t FSE_compress	(	void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize )

FSE_compress() : Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).

Returns

: size of compressed data (<= dstCapacity). Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. if FSE_isError(return), compression failed (more details using FSE_getErrorName())

FSE_compress2()

FSE_PUBLIC_API size_t FSE_compress2	(	void *	dst,
		size_t	dstSize,
		const void *	src,
		size_t	srcSize,
		unsigned	maxSymbolValue,
		unsigned	tableLog )

FSE_compress2() : Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' Both parameters can be defined as '0' to mean : use default value

Returns

: size of compressed data Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. if FSE_isError(return), it's an error code.

FSE_compress_usingCTable()

FSE_PUBLIC_API size_t FSE_compress_usingCTable	(	void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize,
		const FSE_CTable *	ct )

FSE_compress_usingCTable(): Compress src using ct into dst which must be already allocated.

Returns

: size of compressed data (<= dstCapacity), or 0 if compressed data could not fit into dst, or an errorCode, which can be tested using FSE_isError()

FSE_compressBound()

FSE_PUBLIC_API size_t FSE_compressBound

(

size_t

size

)

FSE_createCTable()

FSE_PUBLIC_API FSE_CTable * FSE_createCTable	(	unsigned	maxSymbolValue,
		unsigned	tableLog )

FSE_createDTable()

FSE_PUBLIC_API FSE_DTable * FSE_createDTable

(

unsigned

tableLog

)

FSE_decompress()

static size_t FSE_decompress	(	void *	dst,
		size_t	dstCapacity,
		const void *	cSrc,
		size_t	cSrcSize )

FSE_decompress(): Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', into already allocated destination buffer 'dst', of size 'dstCapacity'.

Returns

: size of regenerated data (<= maxDstSize), or an error code, which can be tested using FSE_isError() .

Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! Why ? : making this distinction requires a header. Header management is intentionally delegated to the user layer, which can better manage special cases.

FSE_decompress_usingDTable()

static size_t FSE_decompress_usingDTable	(	void *	dst,
		size_t	dstCapacity,
		const void *	cSrc,
		size_t	cSrcSize,
		const FSE_DTable *	dt )

FSE_decompress_usingDTable(): Decompress compressed source cSrc of size cSrcSize using dt into dst which must be already allocated.

Returns

: size of regenerated data (necessarily <= dstCapacity), or an errorCode, which can be tested using FSE_isError()

FSE_freeCTable()

FSE_PUBLIC_API void FSE_freeCTable

(

FSE_CTable *

ct

)

FSE_freeDTable()

FSE_PUBLIC_API void FSE_freeDTable

(

FSE_DTable *

dt

)

FSE_getErrorName()

FSE_PUBLIC_API const char * FSE_getErrorName

(

size_t

code

)

FSE_isError()

FSE_PUBLIC_API unsigned FSE_isError

(

size_t

code

)

FSE_NCountWriteBound()

FSE_PUBLIC_API size_t FSE_NCountWriteBound	(	unsigned	maxSymbolValue,
		unsigned	tableLog )

FSE_NCountWriteBound(): Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. Typically useful for allocation purpose.

FSE_normalizeCount()

FSE_PUBLIC_API size_t FSE_normalizeCount	(	short *	normalizedCounter,
		unsigned	tableLog,
		const unsigned *	count,
		size_t	srcSize,
		unsigned	maxSymbolValue,
		unsigned	useLowProbCount )

FSE_normalizeCount(): normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). useLowProbCount is a boolean parameter which trades off compressed size for faster header decoding. When it is set to 1, the compressed data will be slightly smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0 is a good default, since header deserialization makes a big speed difference. Otherwise, useLowProbCount=1 is a good default, since the speed difference is small.

Returns

: tableLog, or an errorCode, which can be tested using FSE_isError()

FSE_optimalTableLog()

FSE_PUBLIC_API unsigned FSE_optimalTableLog	(	unsigned	maxTableLog,
		size_t	srcSize,
		unsigned	maxSymbolValue )

FSE_compress() does the following:

1. count symbol occurrence from source[] into table count[] (see hist.h)
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
3. save normalized counters to memory buffer using writeNCount()
4. build encoding table 'CTable' from normalized counters
5. encode the data stream using encoding table 'CTable'

FSE_decompress() does the following:

1. read normalized counters with readNCount()
2. build decoding table 'DTable' from normalized counters
3. decode the data stream using decoding table 'DTable'

The following API allows targeting specific sub-functions for advanced tasks. For example, it's possible to compress several blocks using the same 'CTable', or to save and provide normalized distribution using external method.

FSE_optimalTableLog(): dynamically downsize 'tableLog' when conditions are met. It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.

Returns

: recommended tableLog (necessarily <= 'maxTableLog')

FSE_readNCount()

FSE_PUBLIC_API size_t FSE_readNCount	(	short *	normalizedCounter,
		unsigned *	maxSymbolValuePtr,
		unsigned *	tableLogPtr,
		const void *	rBuffer,
		size_t	rBuffSize )

Tutorial :

The first step is to count all symbols. FSE_count() does this job very fast. Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. 'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) FSE_count() will return the number of occurrence of the most frequent symbol. This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).

The next step is to normalize the frequencies. FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. It also guarantees a minimum of 1 to any Symbol with frequency >= 1. You can use 'tableLog'==0 to mean "use default tableLog value". If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").

The result of FSE_normalizeCount() will be saved into a table, called 'normalizedCounter', which is a table of signed short. 'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. The return value is tableLog if everything proceeded as expected. It is 0 if there is a single symbol within distribution. If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).

'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). 'buffer' must be already allocated. For guaranteed success, buffer size must be at least FSE_headerBound(). The result of the function is the number of bytes written into 'buffer'. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).

'normalizedCounter' can then be used to create the compression table 'CTable'. The space required by 'CTable' must be already allocated, using FSE_createCTable(). You can then use FSE_buildCTable() to fill 'CTable'. If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).

'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' The function returns the size of compressed data (without header), necessarily <= dstCapacity. If it returns '0', compressed data could not fit into 'dst'. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).

FSE_readNCount(): Read compactly saved 'normalizedCounter' from 'rBuffer'.

Returns

: size read from 'rBuffer', or an errorCode, which can be tested using FSE_isError(). maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values

FSE_readNCount_bmi2()

FSE_PUBLIC_API size_t FSE_readNCount_bmi2	(	short *	normalizedCounter,
		unsigned *	maxSymbolValuePtr,
		unsigned *	tableLogPtr,
		const void *	rBuffer,
		size_t	rBuffSize,
		int	bmi2 )

FSE_readNCount_bmi2(): Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise.

FSE_versionNumber()

FSE_PUBLIC_API unsigned FSE_versionNumber

(

void

)

library version number; to be used when checking dll version

FSE_writeNCount()

FSE_PUBLIC_API size_t FSE_writeNCount	(	void *	buffer,
		size_t	bufferSize,
		const short *	normalizedCounter,
		unsigned	maxSymbolValue,
		unsigned	tableLog )

FSE_writeNCount(): Compactly save 'normalizedCounter' into 'buffer'.

Returns

: size of the compressed table, or an errorCode, which can be tested using FSE_isError().