dibio.c File Reference

#include "platform.h"
#include "util.h"
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>
#include "timefn.h"
#include "../lib/common/debug.h"
#include "../lib/common/mem.h"
#include "../lib/zstd_errors.h"
#include "dibio.h"

Data Structures
struct	fileStats

Macros
#define	KB   *(1 <<10)
#define	MB   *(1 <<20)
#define	GB   *(1U<<30)
#define	SAMPLESIZE_MAX   (128 KB)
#define	MEMMULT   11 /* rough estimation : memory cost to analyze 1 byte of sample */
#define	COVER_MEMMULT   9 /* rough estimation : memory cost to analyze 1 byte of sample */
#define	FASTCOVER_MEMMULT   1 /* rough estimation : memory cost to analyze 1 byte of sample */
#define	NOISELENGTH   32
#define	MAX_SAMPLES_SIZE   (2 GB) /* training dataset limited to 2GB */
#define	DISPLAY(...)
#define	DISPLAYLEVEL(l, ...)
#define	DISPLAYUPDATE(l, ...)
#define	DEBUG   0
#define	DEBUGOUTPUT(...)
#define	EXM_THROW(error, ...)
#define	MIN(a, b)
#define	DiB_rotl32(x, r)

Functions
int	DiB_trainFromFiles (const char *dictFileName, size_t maxDictSize, const char **fileNamesTable, int nbFiles, size_t chunkSize, ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams, ZDICT_fastCover_params_t *fastCoverParams, int optimize, unsigned memLimit)

Macro Definition Documentation

COVER_MEMMULT

#define COVER_MEMMULT   9 /* rough estimation : memory cost to analyze 1 byte of sample */

DEBUG

#define DEBUG   0

DEBUGOUTPUT

#define DEBUGOUTPUT

(

...

)

Value:

if (DEBUG) DISPLAY(__VA_ARGS__);

DiB_rotl32

#define DiB_rotl32	(		x,
			r )

Value:

((x << r) | (x >> (32 - r)))

DISPLAY

#define DISPLAY

(

...

)

Value:

fprintf(stderr, __VA_ARGS__)

DISPLAYLEVEL

#define DISPLAYLEVEL	(		l,
			... )

Value:

if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }

DISPLAYUPDATE

#define DISPLAYUPDATE	(		l,
			... )

Value:

            { if (displayLevel>=l) { \
            if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
            { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
            if (displayLevel>=4) fflush(stderr); } } }

EXM_THROW

#define EXM_THROW	(		error,
			... )

Value:

{                                                                         \
    DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
    DISPLAY("Error %i : ", error);                                        \
    DISPLAY(__VA_ARGS__);                                                 \
    DISPLAY("\n");                                                        \
    exit(error);                                                          \
}

FASTCOVER_MEMMULT

#define FASTCOVER_MEMMULT   1 /* rough estimation : memory cost to analyze 1 byte of sample */

GB

#define GB   *(1U<<30)

KB

#define KB   *(1 <<10)

MAX_SAMPLES_SIZE

#define MAX_SAMPLES_SIZE   (2 GB) /* training dataset limited to 2GB */

MB

#define MB   *(1 <<20)

MEMMULT

#define MEMMULT   11 /* rough estimation : memory cost to analyze 1 byte of sample */

MIN

#define MIN	(		a,
			b )

Value:

((a) < (b) ? (a) : (b))

NOISELENGTH

#define NOISELENGTH   32

SAMPLESIZE_MAX

#define SAMPLESIZE_MAX   (128 KB)

Function Documentation

DiB_trainFromFiles()

int DiB_trainFromFiles	(	const char *	dictFileName,
		size_t	maxDictSize,
		const char **	fileNamesTable,
		int	nbFiles,
		size_t	chunkSize,
		ZDICT_legacy_params_t *	params,
		ZDICT_cover_params_t *	coverParams,
		ZDICT_fastCover_params_t *	fastCoverParams,
		int	optimize,
		unsigned	memLimit )

DiB_trainFromFiles() : Train a dictionary from a set of files provided by fileNamesTable. Resulting dictionary is written into file dictFileName. parameters is optional and can be provided with values set to 0, meaning "default".

Returns

: 0 == ok. Any other : error.