1 /*
2 bench.c - Demo program to benchmark open-source compression algorithm
3 Copyright (C) Yann Collet 2012-2020
4
5 GPL v2 License
6
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
11
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License along
18 with this program; if not, write to the Free Software Foundation, Inc.,
19 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21 You can contact the author at :
22 - LZ4 source repository : https://github.com/lz4/lz4
23 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
24 */
25
26
27 #if defined(_MSC_VER) || defined(_WIN32)
28 /* S_ISREG & gettimeofday() are not supported by MSVC */
29 # define BMK_LEGACY_TIMER 1
30 #endif
31
32
33 /**************************************
34 * Includes
35 **************************************/
36 #include "platform.h" /* _CRT_SECURE_NO_WARNINGS, Large Files support */
37 #include "util.h" /* U32, UTIL_getFileSize */
38 #include <stdlib.h> /* malloc, free */
39 #include <stdio.h> /* fprintf, fopen, ftello */
40 #include <sys/types.h> /* stat64 */
41 #include <sys/stat.h> /* stat64 */
42 #include <string.h> /* strcmp */
43 #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
44
45 #define LZ4_DISABLE_DEPRECATE_WARNINGS /* LZ4_decompress_fast */
46 #include "lz4.h"
47 #include "lz4hc.h"
48 #include "lz4frame.h"
49
50 #include "xxhash.h"
51
52
53 /**************************************
54 * Constants
55 **************************************/
56 #define PROGRAM_DESCRIPTION "LZ4 speed analyzer"
57 #define AUTHOR "Yann Collet"
58 #define WELCOME_MESSAGE "*** %s v%s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, LZ4_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR
59
60 #define NBLOOPS 6
61 #define TIMELOOP (CLOCKS_PER_SEC * 19 / 10)
62
63 #define KB *(1 <<10)
64 #define MB *(1 <<20)
65 #define GB *(1U<<30)
66
67 #define KNUTH 2654435761U
68 #define MAX_MEM (1920 MB)
69 #define DEFAULT_CHUNKSIZE (4 MB)
70
71 #define ALL_COMPRESSORS 0
72 #define ALL_DECOMPRESSORS 0
73
74
75 /**************************************
76 * Local structures
77 **************************************/
78 struct chunkParameters
79 {
80 U32 id;
81 char* origBuffer;
82 char* compressedBuffer;
83 int origSize;
84 int compressedSize;
85 };
86
87
88 /**************************************
89 * Macros
90 **************************************/
91 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
92 #define PROGRESS(...) g_noPrompt ? 0 : DISPLAY(__VA_ARGS__)
93
94
95 /**************************************
96 * Benchmark Parameters
97 **************************************/
98 static int g_chunkSize = DEFAULT_CHUNKSIZE;
99 static int g_nbIterations = NBLOOPS;
100 static int g_pause = 0;
101 static int g_compressionTest = 1;
102 static int g_compressionAlgo = ALL_COMPRESSORS;
103 static int g_decompressionTest = 1;
104 static int g_decompressionAlgo = ALL_DECOMPRESSORS;
105 static int g_noPrompt = 0;
106
BMK_setBlocksize(int bsize)107 static void BMK_setBlocksize(int bsize)
108 {
109 g_chunkSize = bsize;
110 DISPLAY("-Using Block Size of %i KB-\n", g_chunkSize>>10);
111 }
112
BMK_setNbIterations(int nbLoops)113 static void BMK_setNbIterations(int nbLoops)
114 {
115 g_nbIterations = nbLoops;
116 DISPLAY("- %i iterations -\n", g_nbIterations);
117 }
118
BMK_setPause(void)119 static void BMK_setPause(void)
120 {
121 g_pause = 1;
122 }
123
124
125 /*********************************************************
126 * Private functions
127 *********************************************************/
BMK_GetClockSpan(clock_t clockStart)128 static clock_t BMK_GetClockSpan( clock_t clockStart )
129 {
130 return clock() - clockStart; /* works even if overflow; max span ~30 mn */
131 }
132
133
BMK_findMaxMem(U64 requiredMem)134 static size_t BMK_findMaxMem(U64 requiredMem)
135 {
136 size_t step = 64 MB;
137 BYTE* testmem = NULL;
138
139 requiredMem = (((requiredMem >> 26) + 1) << 26);
140 requiredMem += 2*step;
141 if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
142
143 while (!testmem) {
144 if (requiredMem > step) requiredMem -= step;
145 else requiredMem >>= 1;
146 testmem = (BYTE*) malloc ((size_t)requiredMem);
147 }
148 free (testmem);
149
150 /* keep some space available */
151 if (requiredMem > step) requiredMem -= step;
152 else requiredMem >>= 1;
153
154 return (size_t)requiredMem;
155 }
156
157
158 /*********************************************************
159 * Memory management, to test LZ4_USER_MEMORY_FUNCTIONS
160 *********************************************************/
LZ4_malloc(size_t s)161 void* LZ4_malloc(size_t s) { return malloc(s); }
LZ4_calloc(size_t n,size_t s)162 void* LZ4_calloc(size_t n, size_t s) { return calloc(n,s); }
LZ4_free(void * p)163 void LZ4_free(void* p) { free(p); }
164
165
166 /*********************************************************
167 * Benchmark function
168 *********************************************************/
169 static LZ4_stream_t LZ4_stream;
local_LZ4_resetDictT(void)170 static void local_LZ4_resetDictT(void)
171 {
172 void* const r = LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
173 assert(r != NULL); (void)r;
174 }
175
local_LZ4_createStream(void)176 static void local_LZ4_createStream(void)
177 {
178 void* const r = LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
179 assert(r != NULL); (void)r;
180 }
181
182 static void* g_chunk0 = NULL;
183 static size_t g_chunk0Size = 0;
local_LZ4_saveDict_init(void)184 static void local_LZ4_saveDict_init(void)
185 {
186 LZ4_loadDict(&LZ4_stream, (const char*)g_chunk0, (int)g_chunk0Size);
187 }
188
local_LZ4_saveDict(const char * in,char * out,int inSize)189 static int local_LZ4_saveDict(const char* in, char* out, int inSize)
190 {
191 (void)in;
192 return LZ4_saveDict(&LZ4_stream, out, inSize);
193 }
194
local_LZ4_compress_default_large(const char * in,char * out,int inSize)195 static int local_LZ4_compress_default_large(const char* in, char* out, int inSize)
196 {
197 return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize));
198 }
199
local_LZ4_compress_default_small(const char * in,char * out,int inSize)200 static int local_LZ4_compress_default_small(const char* in, char* out, int inSize)
201 {
202 return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1);
203 }
204
local_LZ4_compress_destSize(const char * in,char * out,int inSize)205 static int local_LZ4_compress_destSize(const char* in, char* out, int inSize)
206 {
207 return LZ4_compress_destSize(in, out, &inSize, LZ4_compressBound(inSize)-1);
208 }
209
local_LZ4_compress_fast0(const char * in,char * out,int inSize)210 static int local_LZ4_compress_fast0(const char* in, char* out, int inSize)
211 {
212 return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 0);
213 }
214
local_LZ4_compress_fast1(const char * in,char * out,int inSize)215 static int local_LZ4_compress_fast1(const char* in, char* out, int inSize)
216 {
217 return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 1);
218 }
219
local_LZ4_compress_fast2(const char * in,char * out,int inSize)220 static int local_LZ4_compress_fast2(const char* in, char* out, int inSize)
221 {
222 return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 2);
223 }
224
local_LZ4_compress_fast17(const char * in,char * out,int inSize)225 static int local_LZ4_compress_fast17(const char* in, char* out, int inSize)
226 {
227 return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 17);
228 }
229
local_LZ4_compress_fast_extState0(const char * in,char * out,int inSize)230 static int local_LZ4_compress_fast_extState0(const char* in, char* out, int inSize)
231 {
232 return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0);
233 }
234
local_LZ4_compress_fast_continue0(const char * in,char * out,int inSize)235 static int local_LZ4_compress_fast_continue0(const char* in, char* out, int inSize)
236 {
237 return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0);
238 }
239
240 #ifndef LZ4_DLL_IMPORT
241 #if defined (__cplusplus)
242 extern "C" {
243 #endif
244
245 /* declare hidden function */
246 extern int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
247
248 #if defined (__cplusplus)
249 }
250 #endif
251
local_LZ4_compress_forceDict(const char * in,char * out,int inSize)252 static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
253 {
254 return LZ4_compress_forceExtDict(&LZ4_stream, in, out, inSize);
255 }
256 #endif
257
258
259 /* HC compression functions */
260 LZ4_streamHC_t LZ4_streamHC;
local_LZ4_resetStreamHC(void)261 static void local_LZ4_resetStreamHC(void)
262 {
263 LZ4_initStreamHC(&LZ4_streamHC, sizeof(LZ4_streamHC));
264 }
265
local_LZ4_saveDictHC_init(void)266 static void local_LZ4_saveDictHC_init(void)
267 {
268 LZ4_loadDictHC(&LZ4_streamHC, (const char*)g_chunk0, (int)g_chunk0Size);
269 }
270
local_LZ4_saveDictHC(const char * in,char * out,int inSize)271 static int local_LZ4_saveDictHC(const char* in, char* out, int inSize)
272 {
273 (void)in;
274 return LZ4_saveDictHC(&LZ4_streamHC, out, inSize);
275 }
276
local_LZ4_compress_HC(const char * in,char * out,int inSize)277 static int local_LZ4_compress_HC(const char* in, char* out, int inSize)
278 {
279 return LZ4_compress_HC(in, out, inSize, LZ4_compressBound(inSize), 9);
280 }
281
local_LZ4_compress_HC_extStateHC(const char * in,char * out,int inSize)282 static int local_LZ4_compress_HC_extStateHC(const char* in, char* out, int inSize)
283 {
284 return LZ4_compress_HC_extStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize), 9);
285 }
286
local_LZ4_compress_HC_continue(const char * in,char * out,int inSize)287 static int local_LZ4_compress_HC_continue(const char* in, char* out, int inSize)
288 {
289 return LZ4_compress_HC_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize));
290 }
291
292
293 /* decompression functions */
local_LZ4_decompress_fast(const char * in,char * out,int inSize,int outSize)294 static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
295 {
296 (void)inSize;
297 LZ4_decompress_fast(in, out, outSize);
298 return outSize;
299 }
300
local_LZ4_decompress_fast_usingDict_prefix(const char * in,char * out,int inSize,int outSize)301 static int local_LZ4_decompress_fast_usingDict_prefix(const char* in, char* out, int inSize, int outSize)
302 {
303 (void)inSize;
304 LZ4_decompress_fast_usingDict(in, out, outSize, out - 65536, 65536);
305 return outSize;
306 }
307
local_LZ4_decompress_fast_usingExtDict(const char * in,char * out,int inSize,int outSize)308 static int local_LZ4_decompress_fast_usingExtDict(const char* in, char* out, int inSize, int outSize)
309 {
310 (void)inSize;
311 LZ4_decompress_fast_usingDict(in, out, outSize, out - 65536, 65535);
312 return outSize;
313 }
314
local_LZ4_decompress_safe(const char * in,char * out,int inSize,int outSize)315 static int local_LZ4_decompress_safe(const char* in, char* out, int inSize, int outSize)
316 {
317 return LZ4_decompress_safe(in, out, inSize, outSize);
318 }
319
local_LZ4_decompress_safe_withPrefix64k(const char * in,char * out,int inSize,int outSize)320 static int local_LZ4_decompress_safe_withPrefix64k(const char* in, char* out, int inSize, int outSize)
321 {
322 LZ4_decompress_safe_withPrefix64k(in, out, inSize, outSize);
323 return outSize;
324 }
325
local_LZ4_decompress_safe_usingDict(const char * in,char * out,int inSize,int outSize)326 static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize)
327 {
328 LZ4_decompress_safe_usingDict(in, out, inSize, outSize, out - 65536, 65536);
329 return outSize;
330 }
331
local_LZ4_decompress_safe_partial_usingDict(const char * in,char * out,int inSize,int outSize)332 static int local_LZ4_decompress_safe_partial_usingDict(const char* in, char* out, int inSize, int outSize)
333 {
334 int result = LZ4_decompress_safe_partial_usingDict(in, out, inSize, outSize - 5, outSize, out - 65536, 65536);
335 if (result < 0) return result;
336 return outSize;
337 }
338
339 #ifndef LZ4_DLL_IMPORT
340 #if defined (__cplusplus)
341 extern "C" {
342 #endif
343
344 extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize);
345
346 #if defined (__cplusplus)
347 }
348 #endif
349
local_LZ4_decompress_safe_forceExtDict(const char * in,char * out,int inSize,int outSize)350 static int local_LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize)
351 {
352 LZ4_decompress_safe_forceExtDict(in, out, inSize, outSize, out - 65536, 65536);
353 return outSize;
354 }
355 #endif
356
357 #ifndef LZ4_DLL_IMPORT
358 #if defined (__cplusplus)
359 extern "C" {
360 #endif
361
362 extern int LZ4_decompress_safe_partial_forceExtDict(const char* in, char* out, int inSize, int targetOutputSize, int dstCapacity, const void* dict, size_t dictSize);
363
364 #if defined (__cplusplus)
365 }
366 #endif
367
local_LZ4_decompress_safe_partial_forceExtDict(const char * in,char * out,int inSize,int outSize)368 static int local_LZ4_decompress_safe_partial_forceExtDict(const char* in, char* out, int inSize, int outSize)
369 {
370 int result = LZ4_decompress_safe_partial_forceExtDict(in, out, inSize, outSize - 5, outSize, out - 65536, 65536);
371 if (result < 0) return result;
372 return outSize;
373 }
374 #endif
375
local_LZ4_decompress_safe_partial(const char * in,char * out,int inSize,int outSize)376 static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
377 {
378 int result = LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
379 if (result < 0) return result;
380 return outSize;
381 }
382
383
384 /* frame functions */
local_LZ4F_compressFrame(const char * in,char * out,int inSize)385 static int local_LZ4F_compressFrame(const char* in, char* out, int inSize)
386 {
387 assert(inSize >= 0);
388 return (int)LZ4F_compressFrame(out, LZ4F_compressFrameBound((size_t)inSize, NULL), in, (size_t)inSize, NULL);
389 }
390
391 LZ4F_cctx* g_cctx = NULL;
local_LZ4F_compress(const char * in,char * out,int inSize)392 static int local_LZ4F_compress(const char* in, char* out, int inSize)
393 {
394 /* output buffer size is assumed */
395 size_t const outSize = LZ4F_compressFrameBound((size_t)inSize, NULL);
396 size_t cSize = 0;
397 assert(inSize >= 0);
398 if (g_cctx == NULL) {
399 /* create and initialize LZ4F compression context the first time */
400 LZ4F_createCompressionContext(&g_cctx, LZ4F_VERSION);
401 assert(g_cctx != NULL);
402 } /* re-use existing compression context otherwise */
403 { size_t const cbSize = LZ4F_compressBegin(g_cctx, out, outSize, NULL);
404 assert(!LZ4F_isError(cbSize));
405 cSize += cbSize;
406 }
407 { size_t const cuSize = LZ4F_compressUpdate(g_cctx, out+cSize, outSize-cSize, in, (size_t)inSize, NULL);
408 assert(!LZ4F_isError(cuSize));
409 cSize += cuSize;
410 }
411 { size_t const ceSize = LZ4F_compressEnd(g_cctx, out+cSize, outSize-cSize, NULL);
412 assert(!LZ4F_isError(ceSize));
413 cSize += ceSize;
414 }
415 return (int)cSize;
416 }
417
418 static LZ4F_decompressionContext_t g_dCtx;
419
local_LZ4F_decompress(const char * in,char * out,int inSize,int outSize)420 static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outSize)
421 {
422 size_t srcSize = (size_t)inSize;
423 size_t dstSize = (size_t)outSize;
424 size_t result;
425 assert(inSize >= 0);
426 assert(outSize >= 0);
427 result = LZ4F_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL);
428 if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame \n"); exit(8); }
429 if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect \n"); exit(9); }
430 return (int)dstSize;
431 }
432
local_LZ4F_decompress_followHint(const char * src,char * dst,int srcSize,int dstSize)433 static int local_LZ4F_decompress_followHint(const char* src, char* dst, int srcSize, int dstSize)
434 {
435 size_t totalInSize = (size_t)srcSize;
436 size_t maxOutSize = (size_t)dstSize;
437
438 size_t inPos = 0;
439 size_t inSize = 0;
440 size_t outPos = 0;
441 size_t outRemaining = maxOutSize - outPos;
442
443 for (;;) {
444 size_t const sizeHint =
445 LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
446 assert(!LZ4F_isError(sizeHint));
447
448 inPos += inSize;
449 inSize = sizeHint;
450
451 outPos += outRemaining;
452 outRemaining = maxOutSize - outPos;
453
454 if (!sizeHint) break;
455 }
456
457 /* frame completed */
458 if (inPos != totalInSize) {
459 DISPLAY("Error decompressing frame : must read (%u) full frame (%u) \n",
460 (unsigned)inPos, (unsigned)totalInSize);
461 exit(10);
462 }
463 return (int)outPos;
464
465 }
466
467 /* always provide input by block of 64 KB */
local_LZ4F_decompress_noHint(const char * src,char * dst,int srcSize,int dstSize)468 static int local_LZ4F_decompress_noHint(const char* src, char* dst, int srcSize, int dstSize)
469 {
470 size_t totalInSize = (size_t)srcSize;
471 size_t maxOutSize = (size_t)dstSize;
472
473 size_t inPos = 0;
474 size_t inSize = 64 KB;
475 size_t outPos = 0;
476 size_t outRemaining = maxOutSize - outPos;
477
478 for (;;) {
479 size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
480 assert(!LZ4F_isError(sizeHint));
481
482 inPos += inSize;
483 inSize = (inPos + 64 KB <= totalInSize) ? 64 KB : totalInSize - inPos;
484
485 outPos += outRemaining;
486 outRemaining = maxOutSize - outPos;
487
488 if (!sizeHint) break;
489 }
490
491 /* frame completed */
492 if (inPos != totalInSize) {
493 DISPLAY("Error decompressing frame : must read (%u) full frame (%u) \n",
494 (unsigned)inPos, (unsigned)totalInSize);
495 exit(10);
496 }
497 return (int)outPos;
498
499 }
500
501 typedef struct {
502 const char* name;
503 int (*compressionF)(const char*, char*, int);
504 void (*initFunction)(void);
505 int singleChunk;
506 } CompressionDesc;
507
508 static const CompressionDesc compDescArray[] = {
509 { NULL, NULL, NULL, 0 },
510 { "LZ4_compress_default", local_LZ4_compress_default_large, NULL, 0 },
511 { "LZ4_compress_default(small dst)", local_LZ4_compress_default_small, NULL, 0 },
512 { "LZ4_compress_destSize", local_LZ4_compress_destSize, NULL, 0 },
513 { "LZ4_compress_fast(0)", local_LZ4_compress_fast0, NULL, 0 },
514 { "LZ4_compress_fast(1)", local_LZ4_compress_fast1, NULL, 0 },
515 { "LZ4_compress_fast(2)", local_LZ4_compress_fast2, NULL, 0 },
516 { "LZ4_compress_fast(17)", local_LZ4_compress_fast17, NULL, 0 },
517 { "LZ4_compress_fast_extState(0)", local_LZ4_compress_fast_extState0, NULL, 0 },
518 { "LZ4_compress_fast_continue(0)", local_LZ4_compress_fast_continue0, local_LZ4_createStream, 0 },
519 { "LZ4_compress_HC", local_LZ4_compress_HC, NULL, 0 },
520 { "LZ4_compress_HC_extStateHC", local_LZ4_compress_HC_extStateHC, NULL, 0 },
521 { "LZ4_compress_HC_continue", local_LZ4_compress_HC_continue, local_LZ4_resetStreamHC, 0 },
522 #ifndef LZ4_DLL_IMPORT
523 { "LZ4_compress_forceDict", local_LZ4_compress_forceDict, local_LZ4_resetDictT, 0 },
524 #endif
525 { "LZ4F_compressFrame", local_LZ4F_compressFrame, NULL, 1 },
526 { "LZ4F_compressUpdate", local_LZ4F_compress, NULL, 1 },
527 { "LZ4_saveDict", local_LZ4_saveDict, local_LZ4_saveDict_init, 0 },
528 { "LZ4_saveDictHC", local_LZ4_saveDictHC, local_LZ4_saveDictHC_init, 0 },
529
530 };
531
532 typedef struct {
533 const char* name;
534 int (*decompressionF)(const char*, char*, int, int);
535 int checkResult;
536 int frameFormat;
537 } DecompressionDesc;
538
539 static const DecompressionDesc decDescArray[] = {
540 { NULL, NULL, 0, 0 },
541 { "LZ4_decompress_fast", local_LZ4_decompress_fast, 1, 0 },
542 { "LZ4_decompress_fast_usingDict(prefix)", local_LZ4_decompress_fast_usingDict_prefix, 1, 0 },
543 { "LZ4_decompress_fast_using(Ext)Dict", local_LZ4_decompress_fast_usingExtDict, 1, 0 },
544 { "LZ4_decompress_safe", local_LZ4_decompress_safe, 1, 0 },
545 { "LZ4_decompress_safe_withPrefix64k", local_LZ4_decompress_safe_withPrefix64k, 1, 0 },
546 { "LZ4_decompress_safe_usingDict", local_LZ4_decompress_safe_usingDict, 1, 0 },
547 { "LZ4_decompress_safe_partial", local_LZ4_decompress_safe_partial, 0, 0 },
548 { "LZ4_decompress_safe_partial_usingDict", local_LZ4_decompress_safe_partial_usingDict, 0, 0 },
549 #ifndef LZ4_DLL_IMPORT
550 { "LZ4_decompress_safe_partial_forceExtDict", local_LZ4_decompress_safe_partial_forceExtDict, 0, 0 },
551 { "LZ4_decompress_safe_forceExtDict", local_LZ4_decompress_safe_forceExtDict, 1, 0 },
552 #endif
553 { "LZ4F_decompress", local_LZ4F_decompress, 1, 1 },
554 { "LZ4F_decompLZ4F_decompress_followHintress", local_LZ4F_decompress_followHint, 1, 1 },
555 { "LZ4F_decompress_noHint", local_LZ4F_decompress_noHint, 1, 1 },
556 };
557
558 #define ARRAY_SIZE(a) (sizeof(a) / (sizeof((a)[0])))
559
560 #define NB_COMPRESSION_ALGORITHMS ARRAY_SIZE(compDescArray)
561 #define NB_DECOMPRESSION_ALGORITHMS ARRAY_SIZE(decDescArray)
562
fullSpeedBench(const char ** fileNamesTable,int nbFiles)563 int fullSpeedBench(const char** fileNamesTable, int nbFiles)
564 {
565 int fileIdx=0;
566 clock_t loopDuration = TIMELOOP;
567
568 if (g_nbIterations==0) {
569 loopDuration = CLOCKS_PER_SEC / 50 + 1;
570 g_nbIterations = 1;
571 }
572
573 /* Init */
574 { size_t const errorCode = LZ4F_createDecompressionContext(&g_dCtx, LZ4F_VERSION);
575 if (LZ4F_isError(errorCode)) { DISPLAY("dctx allocation issue \n"); return 10; } }
576
577 /* Loop for each fileName */
578 while (fileIdx<nbFiles) {
579 char* orig_buff = NULL;
580 struct chunkParameters* chunkP = NULL;
581 char* compressed_buff=NULL;
582 const char* const inFileName = fileNamesTable[fileIdx++];
583 FILE* const inFile = fopen( inFileName, "rb" );
584 U64 const inFileSize = UTIL_getFileSize(inFileName);
585 size_t benchedSize = BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */
586 int nbChunks;
587 int maxCompressedChunkSize;
588 size_t readSize;
589 int compressedBuffSize;
590 U32 crcOriginal;
591
592 /* Check infile pre-requisites */
593 if (inFile==NULL) { DISPLAY("Pb opening %s \n", inFileName); return 11; }
594 if (inFileSize==0) { DISPLAY("file is empty \n"); fclose(inFile); return 11; }
595 if (benchedSize==0) { DISPLAY("not enough memory \n"); fclose(inFile); return 11; }
596
597 /* Memory size adjustments */
598 if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
599 if (benchedSize < inFileSize) {
600 DISPLAY("Not enough memory for '%s' full size; testing %i MB only... \n",
601 inFileName, (int)(benchedSize>>20));
602 }
603 if (benchedSize < 8 || g_chunkSize < 8) {
604 DISPLAY(" cannot bench using less then 8 bytes \n");
605 return 1;
606 }
607
608 /* Allocation */
609 nbChunks = (int) ((benchedSize + (size_t)g_chunkSize - 1) / (size_t)g_chunkSize);
610 chunkP = (struct chunkParameters*) malloc((size_t)nbChunks * sizeof(chunkP[0]));
611 orig_buff = (char*) malloc(benchedSize);
612 maxCompressedChunkSize = LZ4_compressBound(g_chunkSize);
613 compressedBuffSize = nbChunks * maxCompressedChunkSize;
614 compressed_buff = (char*)malloc((size_t)compressedBuffSize);
615 if(!chunkP || !orig_buff || !compressed_buff) {
616 DISPLAY("\nError: not enough memory! \n");
617 fclose(inFile);
618 free(orig_buff);
619 free(compressed_buff);
620 free(chunkP);
621 return(12);
622 }
623
624 /* Fill in src buffer */
625 DISPLAY("Loading %s... \r", inFileName);
626 readSize = fread(orig_buff, 1, benchedSize, inFile);
627 fclose(inFile);
628
629 if (readSize != benchedSize) {
630 DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
631 free(orig_buff);
632 free(compressed_buff);
633 free(chunkP);
634 return 13;
635 }
636
637 /* Calculating input Checksum */
638 crcOriginal = XXH32(orig_buff, benchedSize,0);
639
640 /* Bench */
641 { int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb;
642 size_t cSize=0;
643 double ratio=0.;
644
645 DISPLAY("\r%79s\r", "");
646 DISPLAY(" %s : \n", inFileName);
647
648 /* Bench Compression Algorithms */
649 for (cAlgNb=0; (cAlgNb <= (int)NB_COMPRESSION_ALGORITHMS) && (g_compressionTest); cAlgNb++) {
650 const char* compressorName;
651 int (*compressionFunction)(const char*, char*, int);
652 void (*initFunction)(void) = NULL;
653 double bestTime = 100000000.;
654
655 /* filter compressionAlgo only */
656 if ((g_compressionAlgo != ALL_COMPRESSORS) && (g_compressionAlgo != cAlgNb)) continue;
657
658 /* Init data chunks */
659 { int i;
660 size_t remaining = benchedSize;
661 char* in = orig_buff;
662 char* out = compressed_buff;
663 assert(nbChunks >= 1);
664 for (i=0; i<nbChunks; i++) {
665 chunkP[i].id = (U32)i;
666 chunkP[i].origBuffer = in; in += g_chunkSize;
667 assert(g_chunkSize > 0);
668 if (remaining > (size_t)g_chunkSize) {
669 chunkP[i].origSize = g_chunkSize;
670 remaining -= (size_t)g_chunkSize;
671 } else {
672 chunkP[i].origSize = (int)remaining;
673 remaining = 0;
674 }
675 chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
676 chunkP[i].compressedSize = 0;
677 }
678 }
679 g_chunk0 = chunkP[0].origBuffer;
680 g_chunk0Size = (size_t)chunkP[0].origSize;
681
682 if (cAlgNb==0) {
683 DISPLAY("Compression functions : \n");
684 continue;
685 }
686 if (cAlgNb >= (int)NB_COMPRESSION_ALGORITHMS) {
687 continue;
688 }
689 compressorName = compDescArray[cAlgNb].name;
690 compressionFunction = compDescArray[cAlgNb].compressionF;
691 initFunction = compDescArray[cAlgNb].initFunction;
692 if (compDescArray[cAlgNb].singleChunk) {
693 nbChunks=1;
694 chunkP[0].origSize = (int)benchedSize;
695 }
696 if (compressorName==NULL || compressionFunction==NULL) {
697 continue;
698 }
699
700 for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
701 double averageTime;
702 clock_t clockTime;
703
704 PROGRESS("%2i-%-34.34s :%10i ->\r", loopNb, compressorName, (int)benchedSize);
705 { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } /* warming up memory */
706
707 nb_loops = 0;
708 clockTime = clock();
709 while(clock() == clockTime);
710 clockTime = clock();
711 while(BMK_GetClockSpan(clockTime) < loopDuration) {
712 if (initFunction!=NULL) initFunction();
713 for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
714 chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
715 if (chunkP[chunkNb].compressedSize==0) {
716 DISPLAY("ERROR ! %s() = 0 !! \n", compressorName);
717 exit(1);
718 } }
719 nb_loops++;
720 }
721 clockTime = BMK_GetClockSpan(clockTime);
722
723 nb_loops += !nb_loops; /* avoid division by zero */
724 averageTime = ((double)clockTime) / nb_loops / CLOCKS_PER_SEC;
725 if (averageTime < bestTime) bestTime = averageTime;
726 cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += (size_t)chunkP[chunkNb].compressedSize;
727 ratio = (double)cSize/(double)benchedSize*100.;
728 PROGRESS("%2i-%-34.34s :%10i ->%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
729 }
730
731 if (ratio<100.)
732 DISPLAY("%2i-%-34.34s :%10i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
733 else
734 DISPLAY("%2i-%-34.34s :%10i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 100000);
735 }
736
737 /* Prepare layout for decompression */
738 /* Init data chunks */
739 { int i;
740 size_t remaining = benchedSize;
741 char* in = orig_buff;
742 char* out = compressed_buff;
743
744 nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize);
745 for (i=0; i<nbChunks; i++) {
746 chunkP[i].id = (U32)i;
747 chunkP[i].origBuffer = in; in += g_chunkSize;
748 if ((int)remaining > g_chunkSize) {
749 chunkP[i].origSize = g_chunkSize;
750 remaining -= (size_t)g_chunkSize;
751 } else {
752 chunkP[i].origSize = (int)remaining;
753 remaining = 0;
754 }
755 chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
756 chunkP[i].compressedSize = 0;
757 }
758 }
759 for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
760 chunkP[chunkNb].compressedSize = LZ4_compress_default(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize);
761 if (chunkP[chunkNb].compressedSize==0) {
762 DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress");
763 exit(1);
764 } }
765
766 /* Decompression Algorithms */
767 for (dAlgNb=0; (dAlgNb <= (int)NB_DECOMPRESSION_ALGORITHMS) && g_decompressionTest; dAlgNb++) {
768 const char* dName = NULL;
769 int (*decompressionFunction)(const char*, char*, int, int) = NULL;
770 double bestTime = 100000000.;
771 int checkResult = 1;
772
773 if ((g_decompressionAlgo != ALL_DECOMPRESSORS) && (g_decompressionAlgo != dAlgNb)) continue;
774
775 if (dAlgNb==0) { /* just for display */
776 DISPLAY("Decompression functions : \n");
777 continue;
778 }
779 if (dAlgNb >= (int)NB_DECOMPRESSION_ALGORITHMS)
780 continue;
781 dName = decDescArray[dAlgNb].name;
782 decompressionFunction = decDescArray[dAlgNb].decompressionF;
783 if (dName==NULL || decompressionFunction == NULL)
784 continue;
785 checkResult = decDescArray[dAlgNb].checkResult;
786 if (decDescArray[dAlgNb].frameFormat) {
787 /* prepare compressed data using LZ4F frame format */
788 size_t const fcsize = LZ4F_compressFrame(compressed_buff, (size_t)compressedBuffSize, orig_buff, benchedSize, NULL);
789 assert(!LZ4F_isError(fcsize));
790 chunkP[0].origSize = (int)benchedSize;
791 chunkP[0].compressedSize = (int)fcsize;
792 nbChunks = 1;
793 }
794
795 { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing source area, for CRC checking */
796
797 for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
798 double averageTime;
799 clock_t clockTime;
800 U32 crcDecoded;
801
802 PROGRESS("%2i-%-34.34s :%10i ->\r", loopNb, dName, (int)benchedSize);
803
804 nb_loops = 0;
805 clockTime = clock();
806 while(clock() == clockTime);
807 clockTime = clock();
808 while(BMK_GetClockSpan(clockTime) < loopDuration) {
809 for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
810 int const decodedSize = decompressionFunction(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer,
811 chunkP[chunkNb].compressedSize, chunkP[chunkNb].origSize);
812 if (chunkP[chunkNb].origSize != decodedSize) {
813 DISPLAY("ERROR ! %s() == %i != %i !! \n",
814 dName, decodedSize, chunkP[chunkNb].origSize);
815 exit(1);
816 } }
817 nb_loops++;
818 }
819 clockTime = BMK_GetClockSpan(clockTime);
820
821 nb_loops += !nb_loops; /* Avoid division by zero */
822 averageTime = (double)clockTime / nb_loops / CLOCKS_PER_SEC;
823 if (averageTime < bestTime) bestTime = averageTime;
824
825 PROGRESS("%2i-%-34.34s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000);
826
827 /* CRC Checking */
828 crcDecoded = XXH32(orig_buff, benchedSize, 0);
829 if (checkResult && (crcOriginal!=crcDecoded)) {
830 DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n",
831 inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded);
832 exit(1);
833 } }
834
835 DISPLAY("%2i-%-34.34s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000);
836 }
837 }
838 free(orig_buff);
839 free(compressed_buff);
840 free(chunkP);
841 }
842
843 LZ4F_freeDecompressionContext(g_dCtx);
844 if (g_pause) { printf("press enter...\n"); (void)getchar(); }
845
846 return 0;
847 }
848
list(void)849 static int list(void)
850 {
851 size_t n;
852 DISPLAY("Compression Algorithm ids: \n");
853 for (n=0; n<NB_COMPRESSION_ALGORITHMS; n++) {
854 if (compDescArray[n].name) {
855 DISPLAY("%2u: %s \n", (unsigned)n, compDescArray[n].name);
856 }
857 }
858 DISPLAY("Decompression Algorithm ids: \n");
859 for (n=0; n<NB_DECOMPRESSION_ALGORITHMS; n++) {
860 if (decDescArray[n].name) {
861 DISPLAY("%2u: %s \n", (unsigned)n, decDescArray[n].name);
862 }
863 }
864 return 0;
865 }
866
usage(const char * exename)867 static int usage(const char* exename)
868 {
869 DISPLAY( "Usage :\n");
870 DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);
871 DISPLAY( "Arguments :\n");
872 DISPLAY( " -c : compression tests only\n");
873 DISPLAY( " -d : decompression tests only\n");
874 DISPLAY( " -H/-h : Help (this text + advanced options)\n");
875 return 0;
876 }
877
usage_advanced(void)878 static int usage_advanced(void)
879 {
880 DISPLAY( "\nAdvanced options :\n");
881 DISPLAY( " -c# : test only compression function # [1-%i]\n", (int)NB_COMPRESSION_ALGORITHMS);
882 DISPLAY( " -d# : test only decompression function # [1-%i]\n", (int)NB_DECOMPRESSION_ALGORITHMS);
883 DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
884 DISPLAY( " -B# : Block size [4-7](default : 7)\n");
885 return list();
886 }
887
badusage(const char * exename)888 static int badusage(const char* exename)
889 {
890 DISPLAY("Wrong parameters\n");
891 usage(exename);
892 return 0;
893 }
894
main(int argc,const char ** argv)895 int main(int argc, const char** argv)
896 {
897 int i,
898 filenamesStart=2;
899 const char* exename = argv[0];
900 const char* input_filename=0;
901
902 // Welcome message
903 DISPLAY(WELCOME_MESSAGE);
904
905 if (argc<2) { badusage(exename); return 1; }
906
907 for(i=1; i<argc; i++) {
908 const char* argument = argv[i];
909
910 if(!argument) continue; // Protection if argument empty
911 if (!strcmp(argument, "--no-prompt")) {
912 g_noPrompt = 1;
913 continue;
914 }
915
916 // Decode command (note : aggregated commands are allowed)
917 if (argument[0]=='-') {
918 while (argument[1]!=0) {
919 argument ++;
920
921 switch(argument[0])
922 {
923 // Select compression algorithm only
924 case 'c':
925 g_decompressionTest = 0;
926 while ((argument[1]>= '0') && (argument[1]<= '9')) {
927 g_compressionAlgo *= 10;
928 g_compressionAlgo += argument[1] - '0';
929 argument++;
930 }
931 break;
932
933 // Select decompression algorithm only
934 case 'd':
935 g_compressionTest = 0;
936 while ((argument[1]>= '0') && (argument[1]<= '9')) {
937 g_decompressionAlgo *= 10;
938 g_decompressionAlgo += argument[1] - '0';
939 argument++;
940 }
941 break;
942
943 // Display help on usage
944 case 'h' :
945 case 'H': usage(exename); return usage_advanced();
946 case 'l': return list();
947
948 // Modify Block Properties
949 case 'B':
950 while (argument[1]!=0)
951 switch(argument[1])
952 {
953 case '4':
954 case '5':
955 case '6':
956 case '7':
957 { int B = argument[1] - '0';
958 int S = 1 << (8 + 2*B);
959 BMK_setBlocksize(S);
960 argument++;
961 break;
962 }
963 case 'D': argument++; break;
964 default : goto _exit_blockProperties;
965 }
966 _exit_blockProperties:
967 break;
968
969 // Modify Nb Iterations
970 case 'i':
971 if ((argument[1] >='0') && (argument[1] <='9')) {
972 int iters = argument[1] - '0';
973 BMK_setNbIterations(iters);
974 argument++;
975 }
976 break;
977
978 // Pause at the end (hidden option)
979 case 'p': BMK_setPause(); break;
980
981 // Unknown command
982 default : badusage(exename); return 1;
983 }
984 }
985 continue;
986 }
987
988 // first provided filename is input
989 if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
990
991 }
992
993 // No input filename ==> Error
994 if(!input_filename) { badusage(exename); return 1; }
995
996 return fullSpeedBench(argv+filenamesStart, argc-filenamesStart);
997
998 }
999