fse.h File Reference

#include "zstd_deps.h"

Go to the source code of this file.

Macros
#define	FSE_H
#define	FSE_PUBLIC_API
#define	FSE_VERSION_MAJOR   0
#define	FSE_VERSION_MINOR   9
#define	FSE_VERSION_RELEASE   0
#define	FSE_LIB_VERSION   FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
#define	FSE_QUOTE(str)
#define	FSE_EXPAND_AND_QUOTE(str)
#define	FSE_VERSION_STRING   FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
#define	FSE_VERSION_NUMBER   (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)

Functions
FSE_PUBLIC_API unsigned	FSE_versionNumber (void)
FSE_PUBLIC_API size_t	FSE_compressBound (size_t size)
FSE_PUBLIC_API unsigned	FSE_isError (size_t code)
FSE_PUBLIC_API const char *	FSE_getErrorName (size_t code)
FSE_PUBLIC_API unsigned	FSE_optimalTableLog (unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
FSE_PUBLIC_API size_t	FSE_normalizeCount (short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount)
FSE_PUBLIC_API size_t	FSE_NCountWriteBound (unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API size_t	FSE_writeNCount (void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API size_t	FSE_buildCTable (FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
FSE_PUBLIC_API size_t	FSE_compress_usingCTable (void *dst, size_t dstCapacity, const void *src, size_t srcSize, const FSE_CTable *ct)
FSE_PUBLIC_API size_t	FSE_readNCount (short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize)
FSE_PUBLIC_API size_t	FSE_readNCount_bmi2 (short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize, int bmi2)

Macro Definition Documentation

FSE_EXPAND_AND_QUOTE

#define FSE_EXPAND_AND_QUOTE

(

str

)

Value:

FSE_QUOTE(str)

FSE_H

#define FSE_H

FSE_LIB_VERSION

#define FSE_LIB_VERSION   FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE

FSE_PUBLIC_API

#define FSE_PUBLIC_API

FSE_QUOTE

#define FSE_QUOTE

(

str

)

Value:

#str

FSE_VERSION_MAJOR

#define FSE_VERSION_MAJOR   0

FSE_VERSION_MINOR

#define FSE_VERSION_MINOR   9

FSE_VERSION_NUMBER

#define FSE_VERSION_NUMBER   (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)

FSE_VERSION_RELEASE

#define FSE_VERSION_RELEASE   0

FSE_VERSION_STRING

#define FSE_VERSION_STRING   FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)

Function Documentation

FSE_buildCTable()

FSE_PUBLIC_API size_t FSE_buildCTable	(	FSE_CTable *	ct,
		const short *	normalizedCounter,
		unsigned	maxSymbolValue,
		unsigned	tableLog )

FSE_buildCTable(): Builds ct, which must be already allocated, using FSE_createCTable().

Returns

: 0, or an errorCode, which can be tested using FSE_isError()

FSE_compress_usingCTable()

FSE_PUBLIC_API size_t FSE_compress_usingCTable	(	void *	dst,
		size_t	dstCapacity,
		const void *	src,
		size_t	srcSize,
		const FSE_CTable *	ct )

FSE_compress_usingCTable(): Compress src using ct into dst which must be already allocated.

Returns

: size of compressed data (<= dstCapacity), or 0 if compressed data could not fit into dst, or an errorCode, which can be tested using FSE_isError()

FSE_compressBound()

FSE_PUBLIC_API size_t FSE_compressBound

(

size_t

size

)

FSE_getErrorName()

FSE_PUBLIC_API const char * FSE_getErrorName

(

size_t

code

)

FSE_isError()

FSE_PUBLIC_API unsigned FSE_isError

(

size_t

code

)

FSE_NCountWriteBound()

FSE_PUBLIC_API size_t FSE_NCountWriteBound	(	unsigned	maxSymbolValue,
		unsigned	tableLog )

FSE_NCountWriteBound(): Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. Typically useful for allocation purpose.

FSE_normalizeCount()

FSE_PUBLIC_API size_t FSE_normalizeCount	(	short *	normalizedCounter,
		unsigned	tableLog,
		const unsigned *	count,
		size_t	srcSize,
		unsigned	maxSymbolValue,
		unsigned	useLowProbCount )

FSE_normalizeCount(): normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). useLowProbCount is a boolean parameter which trades off compressed size for faster header decoding. When it is set to 1, the compressed data will be slightly smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0 is a good default, since header deserialization makes a big speed difference. Otherwise, useLowProbCount=1 is a good default, since the speed difference is small.

Returns

: tableLog, or an errorCode, which can be tested using FSE_isError()

FSE_optimalTableLog()

FSE_PUBLIC_API unsigned FSE_optimalTableLog	(	unsigned	maxTableLog,
		size_t	srcSize,
		unsigned	maxSymbolValue )

FSE_compress() does the following:

1. count symbol occurrence from source[] into table count[] (see hist.h)
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
3. save normalized counters to memory buffer using writeNCount()
4. build encoding table 'CTable' from normalized counters
5. encode the data stream using encoding table 'CTable'

FSE_decompress() does the following:

1. read normalized counters with readNCount()
2. build decoding table 'DTable' from normalized counters
3. decode the data stream using decoding table 'DTable'

The following API allows targeting specific sub-functions for advanced tasks. For example, it's possible to compress several blocks using the same 'CTable', or to save and provide normalized distribution using external method.

FSE_optimalTableLog(): dynamically downsize 'tableLog' when conditions are met. It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.

Returns

: recommended tableLog (necessarily <= 'maxTableLog')

FSE_readNCount()

FSE_PUBLIC_API size_t FSE_readNCount	(	short *	normalizedCounter,
		unsigned *	maxSymbolValuePtr,
		unsigned *	tableLogPtr,
		const void *	rBuffer,
		size_t	rBuffSize )

Tutorial :

The first step is to count all symbols. FSE_count() does this job very fast. Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. 'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) FSE_count() will return the number of occurrence of the most frequent symbol. This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).

The next step is to normalize the frequencies. FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. It also guarantees a minimum of 1 to any Symbol with frequency >= 1. You can use 'tableLog'==0 to mean "use default tableLog value". If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").

The result of FSE_normalizeCount() will be saved into a table, called 'normalizedCounter', which is a table of signed short. 'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. The return value is tableLog if everything proceeded as expected. It is 0 if there is a single symbol within distribution. If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).

'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). 'buffer' must be already allocated. For guaranteed success, buffer size must be at least FSE_headerBound(). The result of the function is the number of bytes written into 'buffer'. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).

'normalizedCounter' can then be used to create the compression table 'CTable'. The space required by 'CTable' must be already allocated, using FSE_createCTable(). You can then use FSE_buildCTable() to fill 'CTable'. If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).

'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' The function returns the size of compressed data (without header), necessarily <= dstCapacity. If it returns '0', compressed data could not fit into 'dst'. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).

FSE_readNCount(): Read compactly saved 'normalizedCounter' from 'rBuffer'.

Returns

: size read from 'rBuffer', or an errorCode, which can be tested using FSE_isError(). maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values

Tutorial :

The first step is to count all symbols. FSE_count() does this job very fast. Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. 'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) FSE_count() will return the number of occurrence of the most frequent symbol. This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).

The next step is to normalize the frequencies. FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. It also guarantees a minimum of 1 to any Symbol with frequency >= 1. You can use 'tableLog'==0 to mean "use default tableLog value". If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").

The result of FSE_normalizeCount() will be saved into a table, called 'normalizedCounter', which is a table of signed short. 'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. The return value is tableLog if everything proceeded as expected. It is 0 if there is a single symbol within distribution. If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).

'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). 'buffer' must be already allocated. For guaranteed success, buffer size must be at least FSE_headerBound(). The result of the function is the number of bytes written into 'buffer'. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).

'normalizedCounter' can then be used to create the compression table 'CTable'. The space required by 'CTable' must be already allocated, using FSE_createCTable(). You can then use FSE_buildCTable() to fill 'CTable'. If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).

'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' The function returns the size of compressed data (without header), necessarily <= dstCapacity. If it returns '0', compressed data could not fit into 'dst'. If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).

FSE_readNCount(): Read compactly saved 'normalizedCounter' from 'rBuffer'.

Returns

: size read from 'rBuffer', or an errorCode, which can be tested using FSE_isError(). maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values

FSE_readNCount_bmi2()

FSE_PUBLIC_API size_t FSE_readNCount_bmi2	(	short *	normalizedCounter,
		unsigned *	maxSymbolValuePtr,
		unsigned *	tableLogPtr,
		const void *	rBuffer,
		size_t	rBuffSize,
		int	bmi2 )

FSE_readNCount_bmi2(): Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise.

FSE_versionNumber()

FSE_PUBLIC_API unsigned FSE_versionNumber

(

void

)

library version number; to be used when checking dll version

FSE_writeNCount()

FSE_PUBLIC_API size_t FSE_writeNCount	(	void *	buffer,
		size_t	bufferSize,
		const short *	normalizedCounter,
		unsigned	maxSymbolValue,
		unsigned	tableLog )

FSE_writeNCount(): Compactly save 'normalizedCounter' into 'buffer'.

Returns

: size of the compressed table, or an errorCode, which can be tested using FSE_isError().