fastcover.c File Reference

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "../common/mem.h"
#include "../common/pool.h"
#include "../common/threading.h"
#include "../common/zstd_internal.h"
#include "../compress/zstd_compress_internal.h"
#include "../zdict.h"
#include "cover.h"

Data Structures
struct	FASTCOVER_accel_t
struct	FASTCOVER_ctx_t
struct	FASTCOVER_tryParameters_data_s

Macros
#define	ZDICT_STATIC_LINKING_ONLY
#define	FASTCOVER_MAX_SAMPLES_SIZE   (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
#define	FASTCOVER_MAX_F   31
#define	FASTCOVER_MAX_ACCEL   10
#define	FASTCOVER_DEFAULT_SPLITPOINT   0.75
#define	DEFAULT_F   20
#define	DEFAULT_ACCEL   1
#define	DISPLAY(...)
#define	LOCALDISPLAYLEVEL(displayLevel, l, ...)
#define	DISPLAYLEVEL(l, ...)
#define	LOCALDISPLAYUPDATE(displayLevel, l, ...)
#define	DISPLAYUPDATE(l, ...)

Typedefs
typedef struct FASTCOVER_tryParameters_data_s	FASTCOVER_tryParameters_data_t

Functions
ZDICTLIB_STATIC_API size_t	ZDICT_trainFromBuffer_fastCover (void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters)
ZDICTLIB_STATIC_API size_t	ZDICT_optimizeTrainFromBuffer_fastCover (void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t *parameters)

Macro Definition Documentation

DEFAULT_ACCEL

#define DEFAULT_ACCEL   1

DEFAULT_F

#define DEFAULT_F   20

DISPLAY

#define DISPLAY

(

...

)

Value:

  {                                                                            \
    fprintf(stderr, __VA_ARGS__);                                              \
    fflush(stderr);                                                            \
  }

DISPLAYLEVEL

#define DISPLAYLEVEL	(		l,
			... )

Value:

LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)

DISPLAYUPDATE

#define DISPLAYUPDATE	(		l,
			... )

Value:

LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)

FASTCOVER_DEFAULT_SPLITPOINT

#define FASTCOVER_DEFAULT_SPLITPOINT   0.75

FASTCOVER_MAX_ACCEL

#define FASTCOVER_MAX_ACCEL   10

FASTCOVER_MAX_F

#define FASTCOVER_MAX_F   31

FASTCOVER_MAX_SAMPLES_SIZE

#define FASTCOVER_MAX_SAMPLES_SIZE   (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))

There are 32bit indexes used to ref samples, so limit samples size to 4GB on 64bit builds. For 32bit builds we choose 1 GB. Most 32bit platforms have 2GB user-mode addressable space and we allocate a large contiguous buffer, so 1GB is already a high limit.

LOCALDISPLAYLEVEL

#define LOCALDISPLAYLEVEL	(		displayLevel,
			l,
			... )

Value:

  if (displayLevel >= l) {                                                     \
    DISPLAY(__VA_ARGS__);                                                      \
  } /* 0 : no display;   1: errors;   2: default;  3: details;  4: debug */

LOCALDISPLAYUPDATE

#define LOCALDISPLAYUPDATE	(		displayLevel,
			l,
			... )

Value:

  if (displayLevel >= l) {                                                     \
    if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) {             \
      g_time = clock();                                                        \
      DISPLAY(__VA_ARGS__);                                                    \
    }                                                                          \
  }

ZDICT_STATIC_LINKING_ONLY

#define ZDICT_STATIC_LINKING_ONLY

Typedef Documentation

FASTCOVER_tryParameters_data_t

typedef struct FASTCOVER_tryParameters_data_s FASTCOVER_tryParameters_data_t

Parameters for FASTCOVER_tryParameters().

Function Documentation

ZDICT_optimizeTrainFromBuffer_fastCover()

ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover	(	void *	dictBuffer,
		size_t	dictBufferCapacity,
		const void *	samplesBuffer,
		const size_t *	samplesSizes,
		unsigned	nbSamples,
		ZDICT_fastCover_params_t *	parameters )

ZDICT_trainFromBuffer_fastCover()

ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover	(	void *	dictBuffer,
		size_t	dictBufferCapacity,
		const void *	samplesBuffer,
		const size_t *	samplesSizes,
		unsigned	nbSamples,
		ZDICT_fastCover_params_t	parameters )