1 /*
2 fuzzer.c - Fuzzer test tool for LZ4
3 Copyright (C) Yann Collet 2012-2020
4
5 GPL v2 License
6
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
11
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License along
18 with this program; if not, write to the Free Software Foundation, Inc.,
19 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21 You can contact the author at :
22 - LZ4 homepage : http://www.lz4.org
23 - LZ4 source repo : https://github.com/lz4/lz4
24 */
25
26 /*-************************************
27 * Compiler options
28 **************************************/
29 #ifdef _MSC_VER /* Visual Studio */
30 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
31 # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
32 # pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */
33 # pragma warning(disable : 26451) /* disable: C26451: Arithmetic overflow */
34 #endif
35
36
37 /*-************************************
38 * Dependencies
39 **************************************/
40 #if defined(__unix__) && !defined(_AIX) /* must be included before platform.h for MAP_ANONYMOUS */
41 # undef _GNU_SOURCE /* in case it's already defined */
42 # define _GNU_SOURCE /* MAP_ANONYMOUS even in -std=c99 mode */
43 # include <sys/mman.h> /* mmap */
44 #endif
45 #include "platform.h" /* _CRT_SECURE_NO_WARNINGS */
46 #include "util.h" /* U32 */
47 #include <stdlib.h>
48 #include <stdio.h> /* fgets, sscanf */
49 #include <string.h> /* strcmp */
50 #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
51 #include <assert.h>
52 #include <limits.h> /* INT_MAX */
53
54 #if defined(_AIX)
55 # include <sys/mman.h> /* mmap */
56 #endif
57
58 #define LZ4_DISABLE_DEPRECATE_WARNINGS /* LZ4_decompress_fast */
59 #define LZ4_STATIC_LINKING_ONLY
60 #include "lz4.h"
61 #define LZ4_HC_STATIC_LINKING_ONLY
62 #include "lz4hc.h"
63 #define XXH_STATIC_LINKING_ONLY
64 #include "xxhash.h"
65
66
67 /*-************************************
68 * Basic Types
69 **************************************/
70 #if !defined(__cplusplus) && !(defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
71 typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which doesn't need address overflow test) */
72 #endif
73
74
75 /*-************************************
76 * Constants
77 **************************************/
78 #define NB_ATTEMPTS (1<<16)
79 #define COMPRESSIBLE_NOISE_LENGTH (1 << 21)
80 #define FUZ_MAX_BLOCK_SIZE (1 << 17)
81 #define FUZ_MAX_DICT_SIZE (1 << 15)
82 #define FUZ_COMPRESSIBILITY_DEFAULT 60
83 #define PRIME1 2654435761U
84 #define PRIME2 2246822519U
85 #define PRIME3 3266489917U
86
87 #define KB *(1U<<10)
88 #define MB *(1U<<20)
89 #define GB *(1U<<30)
90
91
92 /*-***************************************
93 * Macros
94 *****************************************/
95 #define DISPLAY(...) fprintf(stdout, __VA_ARGS__)
96 #define DISPLAYLEVEL(l, ...) do { if (g_displayLevel>=l) DISPLAY(__VA_ARGS__); } while (0)
97 static int g_displayLevel = 2;
98
99 #define MIN(a,b) ( (a) < (b) ? (a) : (b) )
100
101
102 /*-*******************************************************
103 * Fuzzer functions
104 *********************************************************/
FUZ_GetClockSpan(clock_t clockStart)105 static clock_t FUZ_GetClockSpan(clock_t clockStart)
106 {
107 return clock() - clockStart; /* works even if overflow; max span ~ 30mn */
108 }
109
FUZ_displayUpdate(unsigned testNb)110 static void FUZ_displayUpdate(unsigned testNb)
111 {
112 static clock_t g_time = 0;
113 static const clock_t g_refreshRate = CLOCKS_PER_SEC / 5;
114 if ((FUZ_GetClockSpan(g_time) > g_refreshRate) || (g_displayLevel>=4)) {
115 g_time = clock();
116 DISPLAY("\r%5u ", testNb);
117 fflush(stdout);
118 }
119 }
120
FUZ_rotl32(U32 u32,U32 nbBits)121 static U32 FUZ_rotl32(U32 u32, U32 nbBits)
122 {
123 return ((u32 << nbBits) | (u32 >> (32 - nbBits)));
124 }
125
FUZ_highbit32(U32 v32)126 static U32 FUZ_highbit32(U32 v32)
127 {
128 unsigned nbBits = 0;
129 if (v32==0) return 0;
130 while (v32) { v32 >>= 1; nbBits++; }
131 return nbBits;
132 }
133
FUZ_rand(U32 * src)134 static U32 FUZ_rand(U32* src)
135 {
136 U32 rand32 = *src;
137 rand32 *= PRIME1;
138 rand32 ^= PRIME2;
139 rand32 = FUZ_rotl32(rand32, 13);
140 *src = rand32;
141 return rand32;
142 }
143
144
145 #define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767)
146 #define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
FUZ_fillCompressibleNoiseBuffer(void * buffer,size_t bufferSize,double proba,U32 * seed)147 static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed)
148 {
149 BYTE* const BBuffer = (BYTE*)buffer;
150 size_t pos = 0;
151 U32 const P32 = (U32)(32768 * proba);
152
153 /* First Bytes */
154 while (pos < 20)
155 BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
156
157 while (pos < bufferSize) {
158 /* Select : Literal (noise) or copy (within 64K) */
159 if (FUZ_RAND15BITS < P32) {
160 /* Copy (within 64K) */
161 size_t const length = (size_t)FUZ_RANDLENGTH + 4;
162 size_t const d = MIN(pos+length, bufferSize);
163 size_t match;
164 size_t offset = (size_t)FUZ_RAND15BITS + 1;
165 while (offset > pos) offset >>= 1;
166 match = pos - offset;
167 while (pos < d) BBuffer[pos++] = BBuffer[match++];
168 } else {
169 /* Literal (noise) */
170 size_t const length = FUZ_RANDLENGTH;
171 size_t const d = MIN(pos+length, bufferSize);
172 while (pos < d) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
173 }
174 }
175 }
176
177
178 #define MAX_NB_BUFF_I134 150
179 #define BLOCKSIZE_I134 (32 MB)
180 /*! FUZ_AddressOverflow() :
181 * Aggressively pushes memory allocation limits,
182 * and generates patterns which create address space overflow.
183 * only possible in 32-bits mode */
FUZ_AddressOverflow(void)184 static int FUZ_AddressOverflow(void)
185 {
186 char* buffers[MAX_NB_BUFF_I134+1];
187 int nbBuff=0;
188 int highAddress = 0;
189
190 DISPLAY("Overflow tests : ");
191
192 /* Only possible in 32-bits */
193 if (sizeof(void*)==8) {
194 DISPLAY("64 bits mode : no overflow \n");
195 fflush(stdout);
196 return 0;
197 }
198
199 buffers[0] = (char*)malloc(BLOCKSIZE_I134);
200 buffers[1] = (char*)malloc(BLOCKSIZE_I134);
201 if ((!buffers[0]) || (!buffers[1])) {
202 free(buffers[0]); free(buffers[1]);
203 DISPLAY("not enough memory for tests \n");
204 return 0;
205 }
206
207 for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++) {
208 DISPLAY("%3i \b\b\b\b", nbBuff); fflush(stdout);
209 buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134);
210 if (buffers[nbBuff]==NULL) goto _endOfTests;
211
212 if (((uintptr_t)buffers[nbBuff] > (uintptr_t)0x80000000) && (!highAddress)) {
213 DISPLAY("high address detected : ");
214 fflush(stdout);
215 highAddress=1;
216 }
217
218 { size_t const sizeToGenerateOverflow = (size_t)(- ((uintptr_t)buffers[nbBuff-1]) + 512);
219 int const nbOf255 = (int)((sizeToGenerateOverflow / 255) + 1);
220 char* const input = buffers[nbBuff-1];
221 char* output = buffers[nbBuff];
222 int r;
223 input[0] = (char)0xF0; /* Literal length overflow */
224 input[1] = (char)0xFF;
225 input[2] = (char)0xFF;
226 input[3] = (char)0xFF;
227 { int u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; }
228 r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
229 if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; }
230 input[0] = (char)0x1F; /* Match length overflow */
231 input[1] = (char)0x01;
232 input[2] = (char)0x01;
233 input[3] = (char)0x00;
234 r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
235 if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; }
236
237 output = buffers[nbBuff-2]; /* Reverse in/out pointer order */
238 input[0] = (char)0xF0; /* Literal length overflow */
239 input[1] = (char)0xFF;
240 input[2] = (char)0xFF;
241 input[3] = (char)0xFF;
242 r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
243 if (r>0) goto _overflowError;
244 input[0] = (char)0x1F; /* Match length overflow */
245 input[1] = (char)0x01;
246 input[2] = (char)0x01;
247 input[3] = (char)0x00;
248 r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
249 if (r>0) goto _overflowError;
250 }
251 }
252
253 nbBuff++;
254 _endOfTests:
255 { int i; for (i=0 ; i<nbBuff; i++) free(buffers[i]); }
256 if (!highAddress) DISPLAY("high address not possible \n");
257 else DISPLAY("all overflows correctly detected \n");
258 return 0;
259
260 _overflowError:
261 DISPLAY("Address space overflow error !! \n");
262 exit(1);
263 }
264
265
266 #ifdef __unix__ /* is expected to be triggered on linux+gcc */
267
FUZ_createLowAddr(size_t size)268 static void* FUZ_createLowAddr(size_t size)
269 {
270 void* const lowBuff = mmap((void*)(0x1000), size,
271 PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS,
272 -1, 0);
273 DISPLAYLEVEL(2, "generating low buffer at address %p \n", lowBuff);
274 return lowBuff;
275 }
276
FUZ_freeLowAddr(void * buffer,size_t size)277 static void FUZ_freeLowAddr(void* buffer, size_t size)
278 {
279 if (munmap(buffer, size)) {
280 perror("fuzzer: freeing low address buffer");
281 abort();
282 }
283 }
284
285 #else
286
FUZ_createLowAddr(size_t size)287 static void* FUZ_createLowAddr(size_t size)
288 {
289 return malloc(size);
290 }
291
FUZ_freeLowAddr(void * buffer,size_t size)292 static void FUZ_freeLowAddr(void* buffer, size_t size)
293 {
294 (void)size;
295 free(buffer);
296 }
297
298 #endif
299
300
301 /*! FUZ_findDiff() :
302 * find the first different byte between buff1 and buff2.
303 * presumes buff1 != buff2.
304 * presumes a difference exists before end of either buffer.
305 * Typically invoked after a checksum mismatch.
306 */
FUZ_findDiff(const void * buff1,const void * buff2)307 static void FUZ_findDiff(const void* buff1, const void* buff2)
308 {
309 const BYTE* const b1 = (const BYTE*)buff1;
310 const BYTE* const b2 = (const BYTE*)buff2;
311 size_t u = 0;
312 while (b1[u]==b2[u]) u++;
313 DISPLAY("\nWrong Byte at position %u \n", (unsigned)u);
314 }
315
316
FUZ_test(U32 seed,U32 nbCycles,const U32 startCycle,const double compressibility,U32 duration_s)317 static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration_s)
318 {
319 unsigned long long bytes = 0;
320 unsigned long long cbytes = 0;
321 unsigned long long hcbytes = 0;
322 unsigned long long ccbytes = 0;
323 void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
324 size_t const compressedBufferSize = (size_t)LZ4_compressBound(FUZ_MAX_BLOCK_SIZE);
325 char* const compressedBuffer = (char*)malloc(compressedBufferSize);
326 char* const decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
327 size_t const labSize = 96 KB;
328 void* const lowAddrBuffer = FUZ_createLowAddr(labSize);
329 void* const stateLZ4 = malloc((size_t)LZ4_sizeofState());
330 void* const stateLZ4HC = malloc((size_t)LZ4_sizeofStateHC());
331 LZ4_stream_t LZ4dictBody;
332 LZ4_streamHC_t* const LZ4dictHC = LZ4_createStreamHC();
333 U32 coreRandState = seed;
334 clock_t const clockStart = clock();
335 clock_t const clockDuration = (clock_t)duration_s * CLOCKS_PER_SEC;
336 int result = 0;
337 unsigned cycleNb;
338
339 # define EXIT_MSG(...) do { \
340 printf("Test %u : ", testNb); printf(__VA_ARGS__); \
341 printf(" (seed %u, cycle %u) \n", seed, cycleNb); \
342 exit(1); \
343 } while (0)
344
345 # define FUZ_CHECKTEST(cond, ...) do { if (cond) EXIT_MSG(__VA_ARGS__); } while (0)
346
347 # define FUZ_DISPLAYTEST(...) do { \
348 testNb++; \
349 if (g_displayLevel>=4) { \
350 printf("\r%4u - %2u :", cycleNb, testNb); \
351 printf(" " __VA_ARGS__); \
352 printf(" "); \
353 fflush(stdout); \
354 } \
355 } while (0)
356
357
358 /* init */
359 if(!CNBuffer || !compressedBuffer || !decodedBuffer || !LZ4dictHC) {
360 DISPLAY("Not enough memory to start fuzzer tests");
361 exit(1);
362 }
363 if ( LZ4_initStream(&LZ4dictBody, sizeof(LZ4dictBody)) == NULL) abort();
364 { U32 randState = coreRandState ^ PRIME3;
365 FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
366 }
367
368 /* move to startCycle */
369 for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
370 (void) FUZ_rand(&coreRandState); /* sync coreRandState */
371
372 /* Main test loop */
373 for (cycleNb = startCycle;
374 (cycleNb < nbCycles) || (FUZ_GetClockSpan(clockStart) < clockDuration);
375 cycleNb++) {
376 U32 testNb = 0;
377 U32 randState = FUZ_rand(&coreRandState) ^ PRIME3;
378 int const blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1;
379 int const blockStart = (int)(FUZ_rand(&randState) % (U32)(COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
380 int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
381 int const dictSize = MIN(dictSizeRand, blockStart - 1);
382 int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
383 const char* block = ((char*)CNBuffer) + blockStart;
384 const char* dict = block - dictSize;
385 int compressedSize, HCcompressedSize;
386 int blockContinueCompressedSize;
387 U32 const crcOrig = XXH32(block, (size_t)blockSize, 0);
388 int ret;
389
390 FUZ_displayUpdate(cycleNb);
391
392 /* Compression tests */
393 if ( ((FUZ_rand(&randState) & 63) == 2)
394 && ((size_t)blockSize < labSize) ) {
395 memcpy(lowAddrBuffer, block, blockSize);
396 block = (const char*)lowAddrBuffer;
397 }
398
399 /* Test compression destSize */
400 FUZ_DISPLAYTEST("test LZ4_compress_destSize()");
401 { int cSize, srcSize = blockSize;
402 int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7;
403 char const endCheck = (char)(FUZ_rand(&randState) & 255);
404 compressedBuffer[targetSize] = endCheck;
405 cSize = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize);
406 FUZ_CHECKTEST(cSize > targetSize, "LZ4_compress_destSize() result larger than dst buffer !");
407 FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !");
408 FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() read more than src buffer !");
409 DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", cSize, targetSize, srcSize, blockSize);
410 if (targetSize>0) {
411 /* check correctness */
412 U32 const crcBase = XXH32(block, (size_t)srcSize, 0);
413 char const canary = (char)(FUZ_rand(&randState) & 255);
414 FUZ_CHECKTEST((cSize==0), "LZ4_compress_destSize() compression failed");
415 FUZ_DISPLAYTEST();
416 decodedBuffer[srcSize] = canary;
417 { int const dSize = LZ4_decompress_safe(compressedBuffer, decodedBuffer, cSize, srcSize);
418 FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize");
419 FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
420 }
421 FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !");
422 { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0);
423 FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data");
424 } }
425 DISPLAYLEVEL(5, " OK \n");
426 }
427
428 /* Test compression HC destSize */
429 FUZ_DISPLAYTEST("test LZ4_compress_HC_destSize()");
430 { int cSize, srcSize = blockSize;
431 int const targetSize = srcSize * (int)((FUZ_rand(&randState) & 127)+1) >> 7;
432 char const endCheck = (char)(FUZ_rand(&randState) & 255);
433 void* const ctx = LZ4_createHC(block);
434 FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed");
435 compressedBuffer[targetSize] = endCheck;
436 cSize = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
437 DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ",
438 compressionLevel, cSize, targetSize, srcSize, blockSize);
439 LZ4_freeHC(ctx);
440 FUZ_CHECKTEST(cSize > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
441 FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !");
442 FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !");
443 if (targetSize>0) {
444 /* check correctness */
445 U32 const crcBase = XXH32(block, (size_t)srcSize, 0);
446 char const canary = (char)(FUZ_rand(&randState) & 255);
447 FUZ_CHECKTEST((cSize==0), "LZ4_compress_HC_destSize() compression failed");
448 FUZ_DISPLAYTEST();
449 decodedBuffer[srcSize] = canary;
450 { int const dSize = LZ4_decompress_safe(compressedBuffer, decodedBuffer, cSize, srcSize);
451 FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe failed (%i) on data compressed by LZ4_compressHC_destSize", dSize);
452 FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe failed : decompressed %i bytes, was supposed to decompress %i bytes", dSize, srcSize);
453 }
454 FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe overwrite dst buffer !");
455 { U32 const crcDec = XXH32(decodedBuffer, (size_t)srcSize, 0);
456 FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data");
457 } }
458 DISPLAYLEVEL(5, " OK \n");
459 }
460
461 /* Test compression HC */
462 FUZ_DISPLAYTEST("test LZ4_compress_HC()");
463 HCcompressedSize = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
464 FUZ_CHECKTEST(HCcompressedSize==0, "LZ4_compress_HC() failed");
465
466 /* Test compression HC using external state */
467 FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC()");
468 { int const r = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
469 FUZ_CHECKTEST(r==0, "LZ4_compress_HC_extStateHC() failed");
470 }
471
472 /* Test compression HC using fast reset external state */
473 FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC_fastReset()");
474 { int const r = LZ4_compress_HC_extStateHC_fastReset(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
475 FUZ_CHECKTEST(r==0, "LZ4_compress_HC_extStateHC_fastReset() failed");
476 }
477
478 /* Test compression using external state */
479 FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()");
480 { int const r = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
481 FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState() failed");
482
483 FUZ_DISPLAYTEST("test LZ4_compress_fast_extState() with a too small destination buffer (must fail)");
484 { int const r2 = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, r-1, 8);
485 FUZ_CHECKTEST(r2!=0, "LZ4_compress_fast_extState() should have failed");
486 }
487
488 FUZ_DISPLAYTEST("test LZ4_compress_destSize_extState() with too small dest buffer (must succeed, compress less than full input)");
489 { int inputSize = blockSize;
490 int const r3 = LZ4_compress_destSize_extState(stateLZ4, block, compressedBuffer, &inputSize, r-1, 8);
491 FUZ_CHECKTEST(r3==0, "LZ4_compress_destSize_extState() failed");
492 FUZ_CHECKTEST(inputSize>=blockSize, "LZ4_compress_destSize_extState() should consume less than full input");
493 }
494 }
495
496 /* Test compression using fast reset external state*/
497 FUZ_DISPLAYTEST("test LZ4_compress_fast_extState_fastReset()");
498 { int const r = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
499 FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState_fastReset() failed"); }
500
501 /* Test compression */
502 FUZ_DISPLAYTEST("test LZ4_compress_default()");
503 compressedSize = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
504 FUZ_CHECKTEST(compressedSize<=0, "LZ4_compress_default() failed");
505
506 /* Decompression tests */
507
508 /* Test decompress_fast() with input buffer size exactly correct => must not read out of bound */
509 { char* const cBuffer_exact = (char*)malloc((size_t)compressedSize);
510 assert(cBuffer_exact != NULL);
511 assert(compressedSize <= (int)compressedBufferSize);
512 #if defined(_MSC_VER) && (_MSC_VER <= 1936) /* MSVC 2022 ver 17.6 or earlier */
513 # pragma warning(push)
514 # pragma warning(disable : 6385) /* lz4\tests\fuzzer.c(497): warning C6385: Reading invalid data from 'compressedBuffer'. */
515 #endif
516 memcpy(cBuffer_exact, compressedBuffer, compressedSize);
517 #if defined(_MSC_VER) && (_MSC_VER <= 1936) /* MSVC 2022 ver 17.6 or earlier */
518 # pragma warning(pop)
519 #endif
520
521 /* Test decoding with output size exactly correct => must work */
522 FUZ_DISPLAYTEST("LZ4_decompress_fast() with exact output buffer");
523 { int const r = LZ4_decompress_fast(cBuffer_exact, decodedBuffer, blockSize);
524 FUZ_CHECKTEST(r<0, "LZ4_decompress_fast failed despite correct space");
525 FUZ_CHECKTEST(r!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
526 }
527 { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
528 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
529 }
530
531 /* Test decoding with one byte missing => must fail */
532 FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
533 decodedBuffer[blockSize-1] = 0;
534 { int const r = LZ4_decompress_fast(cBuffer_exact, decodedBuffer, blockSize-1);
535 FUZ_CHECKTEST(r>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
536 }
537 FUZ_CHECKTEST(decodedBuffer[blockSize-1]!=0, "LZ4_decompress_fast overrun specified output buffer");
538
539 /* Test decoding with one byte too much => must fail */
540 FUZ_DISPLAYTEST();
541 { int const r = LZ4_decompress_fast(cBuffer_exact, decodedBuffer, blockSize+1);
542 FUZ_CHECKTEST(r>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
543 }
544
545 /* Test decoding with output size exactly what's necessary => must work */
546 FUZ_DISPLAYTEST();
547 decodedBuffer[blockSize] = 0;
548 { int const r = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize);
549 FUZ_CHECKTEST(r<0, "LZ4_decompress_safe failed despite sufficient space");
550 FUZ_CHECKTEST(r!=blockSize, "LZ4_decompress_safe did not regenerate original data");
551 }
552 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
553 { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
554 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
555 }
556
557 /* Test decoding with more than enough output size => must work */
558 FUZ_DISPLAYTEST();
559 decodedBuffer[blockSize] = 0;
560 decodedBuffer[blockSize+1] = 0;
561 { int const r = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize+1);
562 FUZ_CHECKTEST(r<0, "LZ4_decompress_safe failed despite amply sufficient space");
563 FUZ_CHECKTEST(r!=blockSize, "LZ4_decompress_safe did not regenerate original data");
564 }
565 FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
566 { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
567 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
568 }
569
570 /* Test decoding with output size being one byte too short => must fail */
571 FUZ_DISPLAYTEST();
572 decodedBuffer[blockSize-1] = 0;
573 { int const r = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize-1);
574 FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
575 }
576 FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
577
578 /* Test decoding with output size being 10 bytes too short => must fail */
579 FUZ_DISPLAYTEST();
580 if (blockSize>10) {
581 decodedBuffer[blockSize-10] = 0;
582 { int const r = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize-10);
583 FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short");
584 }
585 FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size");
586 }
587
588 /* noisy src decompression test */
589
590 /* insert noise into src */
591 { U32 const maxNbBits = FUZ_highbit32((U32)compressedSize);
592 size_t pos = 0;
593 for (;;) {
594 /* keep some original src */
595 { U32 const nbBits = FUZ_rand(&randState) % maxNbBits;
596 size_t const mask = (1ULL <<nbBits) - 1;
597 size_t const skipLength = FUZ_rand(&randState) & mask;
598 pos += skipLength;
599 }
600 if (pos >= (size_t)compressedSize) break;
601 /* add noise */
602 { U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits;
603 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
604 size_t const mask = (1ULL <<nbBits) - 1;
605 size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1;
606 size_t const noiseLength = MIN(rNoiseLength, (size_t)compressedSize-pos);
607 size_t const noiseStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - noiseLength);
608 memcpy(cBuffer_exact + pos, (const char*)CNBuffer + noiseStart, noiseLength);
609 pos += noiseLength;
610 } } }
611
612 /* decompress noisy source */
613 FUZ_DISPLAYTEST("decompress noisy source ");
614 { U32 const endMark = 0xA9B1C3D6;
615 memcpy(decodedBuffer+blockSize, &endMark, sizeof(endMark));
616 { int const decompressResult = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize);
617 /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
618 FUZ_CHECKTEST(decompressResult > blockSize, "LZ4_decompress_safe on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)blockSize);
619 }
620 { U32 endCheck; memcpy(&endCheck, decodedBuffer+blockSize, sizeof(endCheck));
621 FUZ_CHECKTEST(endMark!=endCheck, "LZ4_decompress_safe on noisy src : dst buffer overflow");
622 } } /* noisy src decompression test */
623
624 free(cBuffer_exact);
625 }
626
627 /* Test decoding with input size being one byte too short => must fail */
628 FUZ_DISPLAYTEST();
629 { int const r = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
630 FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, result=%i, compressedSize=%i)", blockSize, r, compressedSize);
631 }
632
633 /* Test decoding with input size being one byte too large => must fail */
634 FUZ_DISPLAYTEST();
635 decodedBuffer[blockSize] = 0;
636 { int const r = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
637 FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
638 }
639 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
640
641 /* Test partial decoding => must work */
642 FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial");
643 { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
644 int const targetSize = (int)((size_t)blockSize - missingOutBytes);
645 size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
646 int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
647 char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
648 int const decResult = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize);
649 FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial failed despite valid input data (error:%i)", decResult);
650 FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
651 FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
652 FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial: corruption detected in regenerated data");
653 }
654
655 /* Partial decompression using dictionary. */
656 FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict using no dict");
657 { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
658 int const targetSize = (int)((size_t)blockSize - missingOutBytes);
659 size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
660 int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
661 char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
662 int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, NULL, 0);
663 FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
664 FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
665 FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
666 FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
667 }
668
669 FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict() using prefix as dict");
670 { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
671 int const targetSize = (int)((size_t)blockSize - missingOutBytes);
672 size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
673 int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
674 char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
675 int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, decodedBuffer, dictSize);
676 FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
677 FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
678 FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
679 FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
680 }
681
682 FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict() using external dict");
683 { size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
684 int const targetSize = (int)((size_t)blockSize - missingOutBytes);
685 size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
686 int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
687 char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
688 int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, dict, dictSize);
689 FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
690 FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
691 FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
692 FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
693 }
694
695 /* Test Compression with limited output size */
696
697 /* Test compression with output size being exactly what's necessary (should work) */
698 FUZ_DISPLAYTEST("test LZ4_compress_default() with output buffer just the right size");
699 ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize);
700 FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed despite sufficient space");
701
702 /* Test compression with output size being exactly what's necessary and external state (should work) */
703 FUZ_DISPLAYTEST("test LZ4_compress_fast_extState() with output buffer just the right size");
704 ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedSize, 1);
705 FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed despite sufficient space");
706
707 /* Test HC compression with output size being exactly what's necessary (should work) */
708 FUZ_DISPLAYTEST("test LZ4_compress_HC() with output buffer just the right size");
709 ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
710 FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed despite sufficient space");
711
712 /* Test HC compression with output size being exactly what's necessary (should work) */
713 FUZ_DISPLAYTEST("test LZ4_compress_HC_extStateHC() with output buffer just the right size");
714 ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
715 FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed despite sufficient space");
716
717 /* Test compression with missing bytes into output buffer => must fail */
718 FUZ_DISPLAYTEST("test LZ4_compress_default() with output buffer a bit too short");
719 { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1;
720 if (missingBytes >= compressedSize) missingBytes = compressedSize-1;
721 missingBytes += !missingBytes; /* avoid special case missingBytes==0 */
722 compressedBuffer[compressedSize-missingBytes] = 0;
723 { int const cSize = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes);
724 FUZ_CHECKTEST(cSize, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes);
725 }
726 FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_default overran output buffer ! (%i missingBytes)", missingBytes);
727 }
728
729 /* Test HC compression with missing bytes into output buffer => must fail */
730 FUZ_DISPLAYTEST("test LZ4_compress_HC() with output buffer a bit too short");
731 { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1;
732 if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1;
733 missingBytes += !missingBytes; /* avoid special case missingBytes==0 */
734 compressedBuffer[HCcompressedSize-missingBytes] = 0;
735 { int const hcSize = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel);
736 FUZ_CHECKTEST(hcSize, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes);
737 }
738 FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compress_HC overran output buffer ! (%i missingBytes)", missingBytes);
739 }
740
741
742 /*-******************/
743 /* Dictionary tests */
744 /*-******************/
745
746 /* Compress using dictionary */
747 FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary of size %i", dictSize);
748 { LZ4_stream_t LZ4_stream;
749 LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
750 LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1); /* Just to fill hash tables */
751 blockContinueCompressedSize = LZ4_compress_fast_continue (&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
752 FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
753 }
754
755 /* Decompress with dictionary as prefix */
756 FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as prefix");
757 memcpy(decodedBuffer, dict, dictSize);
758 ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
759 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
760 { U32 const crcCheck = XXH32(decodedBuffer+dictSize, (size_t)blockSize, 0);
761 if (crcCheck!=crcOrig) {
762 FUZ_findDiff(block, decodedBuffer);
763 EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
764 } }
765
766 FUZ_DISPLAYTEST("test LZ4_decompress_safe_usingDict()");
767 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
768 FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
769 { U32 const crcCheck = XXH32(decodedBuffer+dictSize, (size_t)blockSize, 0);
770 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
771 }
772
773 /* Compress using External dictionary */
774 FUZ_DISPLAYTEST("test LZ4_compress_fast_continue(), with non-contiguous dictionary");
775 dict -= (size_t)(FUZ_rand(&randState) & 0xF) + 1; /* create space, so now dictionary is an ExtDict */
776 if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
777 LZ4_loadDict(&LZ4dictBody, dict, dictSize);
778 blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
779 FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
780
781 FUZ_DISPLAYTEST("LZ4_compress_fast_continue() with dictionary and output buffer too short by one byte");
782 LZ4_loadDict(&LZ4dictBody, dict, dictSize);
783 ret = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
784 FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
785
786 FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary loaded with LZ4_loadDict()");
787 DISPLAYLEVEL(5, " compress %i bytes from buffer(%p) into dst(%p) using dict(%p) of size %i \n",
788 blockSize, (const void *)block, (void *)decodedBuffer, (const void *)dict, dictSize);
789 LZ4_loadDict(&LZ4dictBody, dict, dictSize);
790 ret = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
791 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
792 FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
793
794 /* Decompress with dictionary as external */
795 FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as extDict");
796 DISPLAYLEVEL(5, " decoding %i bytes from buffer(%p) using dict(%p) of size %i \n",
797 blockSize, (void *)decodedBuffer, (const void *)dict, dictSize);
798 decodedBuffer[blockSize] = 0;
799 ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
800 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
801 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
802 { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
803 if (crcCheck!=crcOrig) {
804 FUZ_findDiff(block, decodedBuffer);
805 EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
806 } }
807
808 FUZ_DISPLAYTEST();
809 decodedBuffer[blockSize] = 0;
810 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
811 FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
812 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
813 { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
814 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
815 }
816
817 FUZ_DISPLAYTEST();
818 decodedBuffer[blockSize-1] = 0;
819 ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
820 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
821 FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
822
823 FUZ_DISPLAYTEST();
824 decodedBuffer[blockSize-1] = 0;
825 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
826 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
827 FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
828
829 FUZ_DISPLAYTEST();
830 { int const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
831 if (blockSize > missingBytes) {
832 decodedBuffer[blockSize-missingBytes] = 0;
833 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
834 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%i byte)", missingBytes);
835 FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%i byte) (blockSize=%i)", missingBytes, blockSize);
836 } }
837
838 /* Compress using external dictionary stream */
839 { LZ4_stream_t LZ4_stream;
840 int expectedSize;
841 U32 expectedCrc;
842
843 FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_loadDict()");
844 LZ4_loadDict(&LZ4dictBody, dict, dictSize);
845 expectedSize = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
846 FUZ_CHECKTEST(expectedSize<=0, "LZ4_compress_fast_continue reference compression for extDictCtx should have succeeded");
847 expectedCrc = XXH32(compressedBuffer, (size_t)expectedSize, 0);
848
849 FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()");
850 LZ4_loadDict(&LZ4dictBody, dict, dictSize);
851 LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
852 LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
853 blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
854 FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
855
856 /* In the future, it might be desirable to let extDictCtx mode's
857 * output diverge from the output generated by regular extDict mode.
858 * Until that time, this comparison serves as a good regression
859 * test.
860 */
861 FUZ_CHECKTEST(blockContinueCompressedSize != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output (%d expected vs %d actual)", expectedSize, blockContinueCompressedSize);
862 FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)blockContinueCompressedSize, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
863
864 FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short");
865 LZ4_resetStream_fast(&LZ4_stream);
866 LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
867 ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
868 FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
869 /* note : context is no longer dirty after a failed compressed block */
870
871 FUZ_DISPLAYTEST();
872 LZ4_resetStream_fast(&LZ4_stream);
873 LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
874 ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
875 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
876 FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx should work : enough size available within output buffer");
877 FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
878 FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
879
880 FUZ_DISPLAYTEST();
881 LZ4_resetStream_fast(&LZ4_stream);
882 LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
883 ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
884 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
885 FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue using extDictCtx with re-used context should work : enough size available within output buffer");
886 FUZ_CHECKTEST(ret != expectedSize, "LZ4_compress_fast_continue using extDictCtx produced different-sized output");
887 FUZ_CHECKTEST(XXH32(compressedBuffer, (size_t)ret, 0) != expectedCrc, "LZ4_compress_fast_continue using extDictCtx produced different output");
888 }
889
890 /* Decompress with dictionary as external */
891 FUZ_DISPLAYTEST();
892 decodedBuffer[blockSize] = 0;
893 ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
894 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
895 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
896 { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
897 if (crcCheck!=crcOrig) {
898 FUZ_findDiff(block, decodedBuffer);
899 EXIT_MSG("LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
900 } }
901
902 FUZ_DISPLAYTEST();
903 decodedBuffer[blockSize] = 0;
904 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
905 FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
906 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
907 { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
908 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
909 }
910
911 FUZ_DISPLAYTEST();
912 decodedBuffer[blockSize-1] = 0;
913 ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
914 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
915 FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
916
917 FUZ_DISPLAYTEST();
918 decodedBuffer[blockSize-1] = 0;
919 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
920 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
921 FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
922
923 FUZ_DISPLAYTEST("LZ4_decompress_safe_usingDict with a too small output buffer");
924 { int const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
925 if (blockSize > missingBytes) {
926 decodedBuffer[blockSize-missingBytes] = 0;
927 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
928 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%i byte)", missingBytes);
929 FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%i byte) (blockSize=%i)", missingBytes, blockSize);
930 } }
931
932 /* Compress HC using External dictionary */
933 FUZ_DISPLAYTEST("LZ4_compress_HC_continue with an external dictionary");
934 dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
935 if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
936 LZ4_setCompressionLevel (LZ4dictHC, compressionLevel);
937 LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
938 blockContinueCompressedSize = LZ4_compress_HC_continue(LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
939 FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed");
940 FUZ_CHECKTEST(LZ4dictHC->internal_donotuse.dirty, "Context should be clean");
941
942 FUZ_DISPLAYTEST("LZ4_compress_HC_continue with same external dictionary, but output buffer 1 byte too short");
943 LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
944 ret = LZ4_compress_HC_continue(LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
945 FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (expected %i, but result=%i)", blockContinueCompressedSize, ret);
946 /* note : context is no longer dirty after a failed compressed block */
947
948 FUZ_DISPLAYTEST("LZ4_compress_HC_continue with same external dictionary, and output buffer exactly the right size");
949 LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
950 ret = LZ4_compress_HC_continue(LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
951 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different : ret(%i) != expected(%i)", ret, blockContinueCompressedSize);
952 FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer");
953 FUZ_CHECKTEST(LZ4dictHC->internal_donotuse.dirty, "Context should be clean");
954
955 FUZ_DISPLAYTEST();
956 decodedBuffer[blockSize] = 0;
957 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
958 FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
959 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
960 { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
961 if (crcCheck!=crcOrig) {
962 FUZ_findDiff(block, decodedBuffer);
963 EXIT_MSG("LZ4_decompress_safe_usingDict corrupted decoded data");
964 } }
965
966 /* Compress HC using external dictionary stream */
967 FUZ_DISPLAYTEST();
968 { LZ4_streamHC_t* const LZ4_streamHC = LZ4_createStreamHC();
969
970 LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
971 LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
972 LZ4_setCompressionLevel (LZ4_streamHC, compressionLevel);
973 blockContinueCompressedSize = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
974 FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue with ExtDictCtx failed");
975 FUZ_CHECKTEST(LZ4_streamHC->internal_donotuse.dirty, "Context should be clean");
976
977 FUZ_DISPLAYTEST();
978 LZ4_resetStreamHC_fast (LZ4_streamHC, compressionLevel);
979 LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
980 ret = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
981 FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
982 /* note : context is no longer dirty after a failed compressed block */
983
984 FUZ_DISPLAYTEST();
985 LZ4_resetStreamHC_fast (LZ4_streamHC, compressionLevel);
986 LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
987 ret = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
988 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx size is different (%i != %i)", ret, blockContinueCompressedSize);
989 FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx should work : enough size available within output buffer");
990 FUZ_CHECKTEST(LZ4_streamHC->internal_donotuse.dirty, "Context should be clean");
991
992 FUZ_DISPLAYTEST();
993 LZ4_resetStreamHC_fast (LZ4_streamHC, compressionLevel);
994 LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
995 ret = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
996 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue using ExtDictCtx and fast reset size is different (%i != %i)",
997 ret, blockContinueCompressedSize);
998 FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue using ExtDictCtx and fast reset should work : enough size available within output buffer");
999 FUZ_CHECKTEST(LZ4_streamHC->internal_donotuse.dirty, "Context should be clean");
1000
1001 LZ4_freeStreamHC(LZ4_streamHC);
1002 }
1003
1004 FUZ_DISPLAYTEST();
1005 decodedBuffer[blockSize] = 0;
1006 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
1007 FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
1008 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
1009 { U32 const crcCheck = XXH32(decodedBuffer, (size_t)blockSize, 0);
1010 if (crcCheck!=crcOrig) {
1011 FUZ_findDiff(block, decodedBuffer);
1012 EXIT_MSG("LZ4_decompress_safe_usingDict corrupted decoded data");
1013 } }
1014
1015 /* Compress HC continue destSize */
1016 FUZ_DISPLAYTEST();
1017 { int const availableSpace = (int)(FUZ_rand(&randState) % (U32)blockSize) + 5;
1018 int consumedSize = blockSize;
1019 FUZ_DISPLAYTEST();
1020 LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
1021 LZ4_setCompressionLevel(LZ4dictHC, compressionLevel);
1022 blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
1023 DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i \n",
1024 consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel);
1025 FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed");
1026 FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow");
1027 FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow");
1028
1029 FUZ_DISPLAYTEST();
1030 decodedBuffer[consumedSize] = 0;
1031 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize);
1032 FUZ_CHECKTEST(ret != consumedSize, "LZ4_decompress_safe_usingDict regenerated %i bytes (%i expected)", ret, consumedSize);
1033 FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
1034 { U32 const crcSrc = XXH32(block, (size_t)consumedSize, 0);
1035 U32 const crcDst = XXH32(decodedBuffer, (size_t)consumedSize, 0);
1036 if (crcSrc!=crcDst) {
1037 FUZ_findDiff(block, decodedBuffer);
1038 EXIT_MSG("LZ4_decompress_safe_usingDict corrupted decoded data");
1039 } }
1040 }
1041
1042 /* ***** End of tests *** */
1043 /* Fill stats */
1044 assert(blockSize >= 0);
1045 bytes += (unsigned)blockSize;
1046 assert(compressedSize >= 0);
1047 cbytes += (unsigned)compressedSize;
1048 assert(HCcompressedSize >= 0);
1049 hcbytes += (unsigned)HCcompressedSize;
1050 assert(blockContinueCompressedSize >= 0);
1051 ccbytes += (unsigned)blockContinueCompressedSize;
1052 }
1053
1054 if (nbCycles<=1) nbCycles = cycleNb; /* end by time */
1055 bytes += !bytes; /* avoid division by 0 */
1056 printf("\r%7u /%7u - ", cycleNb, nbCycles);
1057 printf("all tests completed successfully \n");
1058 printf("compression ratio: %0.3f%%\n", (double)cbytes/(double)bytes*100);
1059 printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/(double)bytes*100);
1060 printf("ratio with dict: %0.3f%%\n", (double)ccbytes/(double)bytes*100);
1061
1062 /* release memory */
1063 free(CNBuffer);
1064 free(compressedBuffer);
1065 free(decodedBuffer);
1066 FUZ_freeLowAddr(lowAddrBuffer, labSize);
1067 LZ4_freeStreamHC(LZ4dictHC);
1068 free(stateLZ4);
1069 free(stateLZ4HC);
1070 return result;
1071 }
1072
1073
1074 #define testInputSize (196 KB)
1075 #define testCompressedSize (130 KB)
1076 #define ringBufferSize (8 KB)
1077
FUZ_unitTests(int compressionLevel)1078 static void FUZ_unitTests(int compressionLevel)
1079 {
1080 const unsigned testNb = 0;
1081 const unsigned seed = 0;
1082 const unsigned cycleNb= 0;
1083 char* testInput = (char*)malloc(testInputSize);
1084 char* testCompressed = (char*)malloc(testCompressedSize);
1085 char* testVerify = (char*)malloc(testInputSize);
1086 char ringBuffer[ringBufferSize] = {0};
1087 U32 randState = 1;
1088
1089 /* Init */
1090 if (!testInput || !testCompressed || !testVerify) {
1091 EXIT_MSG("not enough memory for FUZ_unitTests");
1092 }
1093 FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState);
1094
1095 /* 32-bits address space overflow test */
1096 FUZ_AddressOverflow();
1097
1098 /* Test decoding with empty input */
1099 DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input \n");
1100 LZ4_decompress_safe(testCompressed, testVerify, 0, testInputSize);
1101
1102 /* Test decoding with a one byte input */
1103 DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input \n");
1104 { char const tmp = (char)0xFF;
1105 LZ4_decompress_safe(&tmp, testVerify, 1, testInputSize);
1106 }
1107
1108 /* Test decoding shortcut edge case */
1109 DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case \n");
1110 { char tmp[17];
1111 /* 14 bytes of literals, followed by a 14 byte match.
1112 * Should not read beyond the end of the buffer.
1113 * See https://github.com/lz4/lz4/issues/508. */
1114 *tmp = (char)0xEE;
1115 memset(tmp + 1, 0, 14);
1116 tmp[15] = 14;
1117 tmp[16] = 0;
1118 { int const r = LZ4_decompress_safe(tmp, testVerify, sizeof(tmp), testInputSize);
1119 FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail");
1120 } }
1121
1122
1123 /* to be tested with undefined sanitizer */
1124 DISPLAYLEVEL(3, "LZ4_compress_default() with NULL input:");
1125 { int const maxCSize = LZ4_compressBound(0);
1126 int const cSize = LZ4_compress_default(NULL, testCompressed, 0, maxCSize);
1127 FUZ_CHECKTEST(!(cSize==1 && testCompressed[0]==0),
1128 "compressing empty should give byte 0"
1129 " (maxCSize == %i) (cSize == %i) (byte == 0x%02X)",
1130 maxCSize, cSize, testCompressed[0]);
1131 }
1132 DISPLAYLEVEL(3, " OK \n");
1133
1134 DISPLAYLEVEL(3, "LZ4_compress_default() with both NULL input and output:");
1135 { int const cSize = LZ4_compress_default(NULL, NULL, 0, 0);
1136 FUZ_CHECKTEST(cSize != 0,
1137 "compressing into NULL must fail"
1138 " (cSize == %i != 0)", cSize);
1139 }
1140 DISPLAYLEVEL(3, " OK \n");
1141
1142 /* in-place compression test */
1143 DISPLAYLEVEL(3, "in-place compression using LZ4_compress_default() :");
1144 { int const sampleSize = 65 KB;
1145 int const maxCSize = LZ4_COMPRESSBOUND(sampleSize);
1146 int const outSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize);
1147 int const startInputIndex = outSize - sampleSize;
1148 char* const startInput = testCompressed + startInputIndex;
1149 XXH32_hash_t const crcOrig = XXH32(testInput, sampleSize, 0);
1150 int cSize;
1151 assert(outSize < (int)testCompressedSize);
1152 memcpy(startInput, testInput, sampleSize); /* copy at end of buffer */
1153 /* compress in-place */
1154 cSize = LZ4_compress_default(startInput, testCompressed, sampleSize, maxCSize);
1155 assert(cSize != 0); /* ensure compression is successful */
1156 assert(maxCSize < INT_MAX);
1157 assert(cSize <= maxCSize);
1158 /* decompress and verify */
1159 { int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize);
1160 assert(dSize == sampleSize); /* correct size */
1161 { XXH32_hash_t const crcCheck = XXH32(testVerify, (size_t)dSize, 0);
1162 FUZ_CHECKTEST(crcCheck != crcOrig, "LZ4_decompress_safe decompression corruption");
1163 } } }
1164 DISPLAYLEVEL(3, " OK \n");
1165
1166 /* in-place decompression test */
1167 DISPLAYLEVEL(3, "in-place decompression, limit case:");
1168 { int const sampleSize = 65 KB;
1169
1170 FUZ_fillCompressibleNoiseBuffer(testInput, sampleSize, 0.0, &randState);
1171 memset(testInput, 0, 267); /* calculated exactly so that compressedSize == originalSize-1 */
1172
1173 { XXH64_hash_t const crcOrig = XXH64(testInput, sampleSize, 0);
1174 int const cSize = LZ4_compress_default(testInput, testCompressed, sampleSize, testCompressedSize);
1175 assert(cSize == sampleSize - 1); /* worst case for in-place decompression */
1176
1177 { int const bufferSize = LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(sampleSize);
1178 int const startInputIndex = bufferSize - cSize;
1179 char* const startInput = testVerify + startInputIndex;
1180 memcpy(startInput, testCompressed, cSize);
1181
1182 /* decompress and verify */
1183 { int const dSize = LZ4_decompress_safe(startInput, testVerify, cSize, sampleSize);
1184 assert(dSize == sampleSize); /* correct size */
1185 { XXH64_hash_t const crcCheck = XXH64(testVerify, (size_t)dSize, 0);
1186 FUZ_CHECKTEST(crcCheck != crcOrig, "LZ4_decompress_safe decompression corruption");
1187 } } } } }
1188 DISPLAYLEVEL(3, " OK \n");
1189
1190 DISPLAYLEVEL(3, "LZ4_initStream with multiple valid alignments : ");
1191 { typedef struct {
1192 LZ4_stream_t state1;
1193 LZ4_stream_t state2;
1194 char c;
1195 LZ4_stream_t state3;
1196 } shct;
1197 shct* const shc = (shct*)malloc(sizeof(*shc));
1198 assert(shc != NULL);
1199 memset(shc, 0, sizeof(*shc));
1200 DISPLAYLEVEL(4, "state1(%p) state2(%p) state3(%p) LZ4_stream_t size(0x%x): ",
1201 (void*)&(shc->state1), (void*)&(shc->state2), (void*)&(shc->state3), (unsigned)sizeof(LZ4_stream_t));
1202 FUZ_CHECKTEST( LZ4_initStream(&(shc->state1), sizeof(shc->state1)) == NULL, "state1 (%p) failed init", (void*)&(shc->state1) );
1203 FUZ_CHECKTEST( LZ4_initStream(&(shc->state2), sizeof(shc->state2)) == NULL, "state2 (%p) failed init", (void*)&(shc->state2) );
1204 FUZ_CHECKTEST( LZ4_initStream(&(shc->state3), sizeof(shc->state3)) == NULL, "state3 (%p) failed init", (void*)&(shc->state3) );
1205 FUZ_CHECKTEST( LZ4_initStream((char*)&(shc->state1) + 1, sizeof(shc->state1)) != NULL,
1206 "hc1+1 (%p) init must fail, due to bad alignment", (void*)((char*)&(shc->state1) + 1) );
1207 free(shc);
1208 }
1209 DISPLAYLEVEL(3, "all inits OK \n");
1210
1211 /* Allocation test */
1212 { LZ4_stream_t* const statePtr = LZ4_createStream();
1213 FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
1214 LZ4_freeStream(statePtr);
1215 }
1216
1217 /* LZ4 streaming tests */
1218 { LZ4_stream_t streamingState;
1219
1220 /* simple compression test */
1221 LZ4_initStream(&streamingState, sizeof(streamingState));
1222 { int const cs = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1);
1223 FUZ_CHECKTEST(cs==0, "LZ4_compress_fast_continue() compression failed!");
1224 { int const r = LZ4_decompress_safe(testCompressed, testVerify, cs, testCompressedSize);
1225 FUZ_CHECKTEST(r!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed");
1226 } }
1227 { U64 const crcOrig = XXH64(testInput, testCompressedSize, 0);
1228 U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
1229 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
1230 }
1231
1232 /* early saveDict */
1233 DISPLAYLEVEL(3, "saveDict (right after init) : ");
1234 { LZ4_stream_t* const ctx = LZ4_initStream(&streamingState, sizeof(streamingState));
1235 assert(ctx != NULL); /* ensure init is successful */
1236
1237 /* Check access violation with asan */
1238 FUZ_CHECKTEST( LZ4_saveDict(ctx, NULL, 0) != 0,
1239 "LZ4_saveDict() can't save anything into (NULL,0)");
1240
1241 /* Check access violation with asan */
1242 { char tmp_buffer[240] = { 0 };
1243 FUZ_CHECKTEST( LZ4_saveDict(ctx, tmp_buffer, sizeof(tmp_buffer)) != 0,
1244 "LZ4_saveDict() can't save anything since compression hasn't started");
1245 } }
1246 DISPLAYLEVEL(3, "OK \n");
1247
1248 /* ring buffer test */
1249 { XXH64_state_t xxhOrig;
1250 XXH64_state_t xxhNewSafe, xxhNewFast;
1251 LZ4_streamDecode_t decodeStateSafe, decodeStateFast;
1252 const U32 maxMessageSizeLog = 10;
1253 const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
1254 U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
1255 U32 iNext = 0;
1256 U32 rNext = 0;
1257 U32 dNext = 0;
1258 const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
1259
1260 XXH64_reset(&xxhOrig, 0);
1261 XXH64_reset(&xxhNewSafe, 0);
1262 XXH64_reset(&xxhNewFast, 0);
1263 LZ4_resetStream_fast(&streamingState);
1264 LZ4_setStreamDecode(&decodeStateSafe, NULL, 0);
1265 LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
1266
1267 while (iNext + messageSize < testCompressedSize) {
1268 int compressedSize; U64 crcOrig;
1269 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
1270 crcOrig = XXH64_digest(&xxhOrig);
1271
1272 memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
1273 compressedSize = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize, 1);
1274 FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_fast_continue() compression failed");
1275
1276 { int const r = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize);
1277 FUZ_CHECKTEST(r!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed"); }
1278
1279 XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
1280 { U64 const crcNew = XXH64_digest(&xxhNewSafe);
1281 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
1282
1283 { int const r = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize);
1284 FUZ_CHECKTEST(r!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed"); }
1285
1286 XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
1287 { U64 const crcNew = XXH64_digest(&xxhNewFast);
1288 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
1289
1290 /* prepare next message */
1291 iNext += messageSize;
1292 rNext += messageSize;
1293 dNext += messageSize;
1294 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
1295 if (rNext + messageSize > ringBufferSize) rNext = 0;
1296 if (dNext + messageSize > dBufferSize) dNext = 0;
1297 } }
1298 }
1299
1300 DISPLAYLEVEL(3, "LZ4_initStreamHC with multiple valid alignments : ");
1301 { typedef struct {
1302 LZ4_streamHC_t hc1;
1303 LZ4_streamHC_t hc2;
1304 char c;
1305 LZ4_streamHC_t hc3;
1306 } shct;
1307 shct* const shc = (shct*)malloc(sizeof(*shc));
1308 assert(shc != NULL);
1309 memset(shc, 0, sizeof(*shc));
1310 DISPLAYLEVEL(4, "hc1(%p) hc2(%p) hc3(%p) size(0x%x): ",
1311 (void*)&(shc->hc1), (void*)&(shc->hc2), (void*)&(shc->hc3),
1312 (unsigned)sizeof(LZ4_streamHC_t));
1313 FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc1), sizeof(shc->hc1)) == NULL, "hc1 (%p) failed init", (void*)&(shc->hc1) );
1314 FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc2), sizeof(shc->hc2)) == NULL, "hc2 (%p) failed init", (void*)&(shc->hc2) );
1315 FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc3), sizeof(shc->hc3)) == NULL, "hc3 (%p) failed init", (void*)&(shc->hc3) );
1316 FUZ_CHECKTEST( LZ4_initStreamHC((char*)&(shc->hc1) + 1, sizeof(shc->hc1)) != NULL,
1317 "hc1+1 (%p) init must fail, due to bad alignment", (void*)((char*)&(shc->hc1) + 1) );
1318 free(shc);
1319 }
1320 DISPLAYLEVEL(3, "all inits OK \n");
1321
1322 /* LZ4 HC streaming tests */
1323 { LZ4_streamHC_t sHC; /* statically allocated */
1324 int result;
1325 LZ4_initStreamHC(&sHC, sizeof(sHC));
1326
1327 /* Allocation test */
1328 DISPLAYLEVEL(3, "Basic HC allocation : ");
1329 { LZ4_streamHC_t* const sp = LZ4_createStreamHC();
1330 FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed");
1331 LZ4_freeStreamHC(sp);
1332 }
1333 DISPLAYLEVEL(3, "OK \n");
1334
1335 /* simple HC compression test */
1336 DISPLAYLEVEL(3, "Simple HC round-trip : ");
1337 { U64 const crc64 = XXH64(testInput, testCompressedSize, 0);
1338 LZ4_setCompressionLevel(&sHC, compressionLevel);
1339 result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
1340 FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
1341 FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
1342
1343 result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize);
1344 FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed");
1345 { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
1346 FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() decompression corruption");
1347 } }
1348 DISPLAYLEVEL(3, "OK \n");
1349
1350 /* saveDictHC test #926 */
1351 DISPLAYLEVEL(3, "saveDictHC test #926 : ");
1352 { LZ4_streamHC_t* const ctx = LZ4_initStreamHC(&sHC, sizeof(sHC));
1353 assert(ctx != NULL); /* ensure init is successful */
1354
1355 /* Check access violation with asan */
1356 FUZ_CHECKTEST( LZ4_saveDictHC(ctx, NULL, 0) != 0,
1357 "LZ4_saveDictHC() can't save anything into (NULL,0)");
1358
1359 /* Check access violation with asan */
1360 { char tmp_buffer[240] = { 0 };
1361 FUZ_CHECKTEST( LZ4_saveDictHC(ctx, tmp_buffer, sizeof(tmp_buffer)) != 0,
1362 "LZ4_saveDictHC() can't save anything since compression hasn't started");
1363 } }
1364 DISPLAYLEVEL(3, "OK \n");
1365
1366 /* long sequence test */
1367 DISPLAYLEVEL(3, "Long sequence HC_destSize test : ");
1368 { size_t const blockSize = 1 MB;
1369 size_t const targetSize = 4116; /* size carefully selected to trigger an overflow */
1370 void* const block = malloc(blockSize);
1371 void* const dstBlock = malloc(targetSize+1);
1372 BYTE const sentinel = 101;
1373 int srcSize;
1374
1375 assert(block != NULL); assert(dstBlock != NULL);
1376 memset(block, 0, blockSize);
1377 ((char*)dstBlock)[targetSize] = sentinel;
1378
1379 LZ4_resetStreamHC_fast(&sHC, 3);
1380 assert(blockSize < INT_MAX);
1381 srcSize = (int)blockSize;
1382 assert(targetSize < INT_MAX);
1383 result = LZ4_compress_HC_destSize(&sHC, (const char*)block, (char*)dstBlock, &srcSize, (int)targetSize, 3);
1384 DISPLAYLEVEL(4, "cSize=%i; readSize=%i; ", result, srcSize);
1385 FUZ_CHECKTEST(result != 4116, "LZ4_compress_HC_destSize() : "
1386 "compression (%i->%i) must fill dstBuffer (%i) exactly",
1387 srcSize, result, (int)targetSize);
1388 FUZ_CHECKTEST(((char*)dstBlock)[targetSize] != sentinel,
1389 "LZ4_compress_HC_destSize() overwrites dst buffer");
1390 FUZ_CHECKTEST(srcSize < 1045000, "LZ4_compress_HC_destSize() doesn't compress enough"
1391 " (%i -> %i , expected > %i)", srcSize, result, 1045000);
1392
1393 LZ4_resetStreamHC_fast(&sHC, 3); /* make sure the context is clean after the test */
1394 free(block);
1395 free(dstBlock);
1396 }
1397 DISPLAYLEVEL(3, " OK \n");
1398
1399 /* simple dictionary HC compression test */
1400 DISPLAYLEVEL(3, "HC dictionary compression test : ");
1401 { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0);
1402 LZ4_resetStreamHC_fast(&sHC, compressionLevel);
1403 LZ4_loadDictHC(&sHC, testInput, 64 KB);
1404 { int const cSize = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
1405 FUZ_CHECKTEST(cSize==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : @return = %i", cSize);
1406 FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
1407 { int const dSize = LZ4_decompress_safe_usingDict(testCompressed, testVerify, cSize, testCompressedSize, testInput, 64 KB);
1408 FUZ_CHECKTEST(dSize!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed");
1409 } }
1410 { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
1411 FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption");
1412 } }
1413 DISPLAYLEVEL(3, " OK \n");
1414
1415 /* multiple HC compression test with dictionary */
1416 { int result1, result2;
1417 int segSize = testCompressedSize / 2;
1418 XXH64_hash_t const crc64 = ( (void)assert((unsigned)segSize + testCompressedSize < testInputSize) ,
1419 XXH64(testInput + segSize, testCompressedSize, 0) );
1420 LZ4_resetStreamHC_fast(&sHC, compressionLevel);
1421 LZ4_loadDictHC(&sHC, testInput, segSize);
1422 result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1);
1423 FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1);
1424 FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
1425 result2 = LZ4_compress_HC_continue(&sHC, testInput + 2*(size_t)segSize, testCompressed+result1, segSize, segSize-1);
1426 FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result2);
1427 FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
1428
1429 result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize);
1430 FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed");
1431 result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize);
1432 FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed");
1433 { XXH64_hash_t const crcNew = XXH64(testVerify, testCompressedSize, 0);
1434 FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
1435 } }
1436
1437 /* remote dictionary HC compression test */
1438 { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0);
1439 LZ4_resetStreamHC_fast(&sHC, compressionLevel);
1440 LZ4_loadDictHC(&sHC, testInput, 32 KB);
1441 result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
1442 FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result);
1443 FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
1444
1445 result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB);
1446 FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test");
1447 { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
1448 FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption");
1449 } }
1450
1451 /* multiple HC compression with ext. dictionary */
1452 { XXH64_state_t crcOrigState;
1453 XXH64_state_t crcNewState;
1454 const char* dict = testInput + 3;
1455 size_t dictSize = (FUZ_rand(&randState) & 8191);
1456 char* dst = testVerify;
1457
1458 size_t segStart = dictSize + 7;
1459 size_t segSize = (FUZ_rand(&randState) & 8191);
1460 int segNb = 1;
1461
1462 LZ4_resetStreamHC_fast(&sHC, compressionLevel);
1463 LZ4_loadDictHC(&sHC, dict, (int)dictSize);
1464
1465 XXH64_reset(&crcOrigState, 0);
1466 XXH64_reset(&crcNewState, 0);
1467
1468 while (segStart + segSize < testInputSize) {
1469 XXH64_hash_t crcOrig;
1470 XXH64_update(&crcOrigState, testInput + segStart, segSize);
1471 crcOrig = XXH64_digest(&crcOrigState);
1472 assert(segSize <= INT_MAX);
1473 result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, (int)segSize, LZ4_compressBound((int)segSize));
1474 FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
1475 FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
1476
1477 result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, (int)segSize, dict, (int)dictSize);
1478 FUZ_CHECKTEST(result!=(int)segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", (int)segNb);
1479 XXH64_update(&crcNewState, dst, segSize);
1480 { U64 const crcNew = XXH64_digest(&crcNewState);
1481 if (crcOrig != crcNew) FUZ_findDiff(dst, testInput+segStart);
1482 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb);
1483 }
1484
1485 dict = dst;
1486 dictSize = segSize;
1487
1488 dst += segSize + 1;
1489 segNb ++;
1490
1491 segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1;
1492 segSize = (FUZ_rand(&randState) & 8191);
1493 } }
1494
1495 /* ring buffer test */
1496 { XXH64_state_t xxhOrig;
1497 XXH64_state_t xxhNewSafe, xxhNewFast;
1498 LZ4_streamDecode_t decodeStateSafe, decodeStateFast;
1499 const U32 maxMessageSizeLog = 10;
1500 const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
1501 U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
1502 U32 iNext = 0;
1503 U32 rNext = 0;
1504 U32 dNext = 0;
1505 const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
1506
1507 XXH64_reset(&xxhOrig, 0);
1508 XXH64_reset(&xxhNewSafe, 0);
1509 XXH64_reset(&xxhNewFast, 0);
1510 LZ4_resetStreamHC_fast(&sHC, compressionLevel);
1511 LZ4_setStreamDecode(&decodeStateSafe, NULL, 0);
1512 LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
1513
1514 while (iNext + messageSize < testCompressedSize) {
1515 int compressedSize;
1516 XXH64_hash_t crcOrig;
1517 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
1518 crcOrig = XXH64_digest(&xxhOrig);
1519
1520 memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
1521 assert(messageSize < INT_MAX);
1522 compressedSize = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, (int)messageSize, testCompressedSize-ringBufferSize);
1523 FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
1524 FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
1525
1526 assert(messageSize < INT_MAX);
1527 result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, testVerify + dNext, compressedSize, (int)messageSize);
1528 FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe_continue() test failed");
1529
1530 XXH64_update(&xxhNewSafe, testVerify + dNext, messageSize);
1531 { XXH64_hash_t const crcNew = XXH64_digest(&xxhNewSafe);
1532 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
1533
1534 assert(messageSize < INT_MAX);
1535 result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, testVerify + dNext, (int)messageSize);
1536 FUZ_CHECKTEST(result!=compressedSize, "ringBuffer : LZ4_decompress_fast_continue() test failed");
1537
1538 XXH64_update(&xxhNewFast, testVerify + dNext, messageSize);
1539 { XXH64_hash_t const crcNew = XXH64_digest(&xxhNewFast);
1540 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
1541
1542 /* prepare next message */
1543 iNext += messageSize;
1544 rNext += messageSize;
1545 dNext += messageSize;
1546 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
1547 if (rNext + messageSize > ringBufferSize) rNext = 0;
1548 if (dNext + messageSize > dBufferSize) dNext = 0;
1549 }
1550 }
1551
1552 /* Ring buffer test : Non synchronized decoder */
1553 /* This test uses minimum amount of memory required to setup a decoding ring buffer
1554 * while being unsynchronized with encoder
1555 * (no assumption done on how the data is encoded, it just follows LZ4 format specification).
1556 * This size is documented in lz4.h, and is LZ4_decoderRingBufferSize(maxBlockSize).
1557 */
1558 { XXH64_state_t xxhOrig;
1559 XXH64_state_t xxhNewSafe, xxhNewFast;
1560 XXH64_hash_t crcOrig;
1561 LZ4_streamDecode_t decodeStateSafe, decodeStateFast;
1562 const int maxMessageSizeLog = 12;
1563 const int maxMessageSize = 1 << maxMessageSizeLog;
1564 const int maxMessageSizeMask = maxMessageSize - 1;
1565 int messageSize;
1566 U32 totalMessageSize = 0;
1567 const int dBufferSize = LZ4_decoderRingBufferSize(maxMessageSize);
1568 char* const ringBufferSafe = testVerify;
1569 char* const ringBufferFast = testVerify + dBufferSize + 1; /* used by LZ4_decompress_fast_continue */
1570 int iNext = 0;
1571 int dNext = 0;
1572 int compressedSize;
1573
1574 assert((size_t)dBufferSize * 2 + 1 < testInputSize); /* space used by ringBufferSafe and ringBufferFast */
1575 XXH64_reset(&xxhOrig, 0);
1576 XXH64_reset(&xxhNewSafe, 0);
1577 XXH64_reset(&xxhNewFast, 0);
1578 LZ4_resetStreamHC_fast(&sHC, compressionLevel);
1579 LZ4_setStreamDecode(&decodeStateSafe, NULL, 0);
1580 LZ4_setStreamDecode(&decodeStateFast, NULL, 0);
1581
1582 #define BSIZE1 (dBufferSize - (maxMessageSize-1))
1583
1584 /* first block */
1585 messageSize = BSIZE1; /* note : we cheat a bit here, in theory no message should be > maxMessageSize. We just want to fill the decoding ring buffer once. */
1586 XXH64_update(&xxhOrig, testInput + iNext, (size_t)messageSize);
1587 crcOrig = XXH64_digest(&xxhOrig);
1588
1589 compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
1590 FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
1591 FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
1592
1593 result = LZ4_decompress_safe_continue(&decodeStateSafe, testCompressed, ringBufferSafe + dNext, compressedSize, messageSize);
1594 FUZ_CHECKTEST(result!=messageSize, "64K D.ringBuffer : LZ4_decompress_safe_continue() test failed");
1595
1596 XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, (size_t)messageSize);
1597 { U64 const crcNew = XXH64_digest(&xxhNewSafe);
1598 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption"); }
1599
1600 result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize);
1601 FUZ_CHECKTEST(result!=compressedSize, "64K D.ringBuffer : LZ4_decompress_fast_continue() test failed");
1602
1603 XXH64_update(&xxhNewFast, ringBufferFast + dNext, (size_t)messageSize);
1604 { U64 const crcNew = XXH64_digest(&xxhNewFast);
1605 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption"); }
1606
1607 /* prepare second message */
1608 dNext += messageSize;
1609 assert(messageSize >= 0);
1610 totalMessageSize += (unsigned)messageSize;
1611 messageSize = maxMessageSize;
1612 iNext = BSIZE1+1;
1613 assert(BSIZE1 >= 65535);
1614 memcpy(testInput + iNext, testInput + (BSIZE1-65535), messageSize); /* will generate a match at max distance == 65535 */
1615 FUZ_CHECKTEST(dNext+messageSize <= dBufferSize, "Ring buffer test : second message should require restarting from beginning");
1616 dNext = 0;
1617
1618 while (totalMessageSize < 9 MB) {
1619 XXH64_update(&xxhOrig, testInput + iNext, (size_t)messageSize);
1620 crcOrig = XXH64_digest(&xxhOrig);
1621
1622 compressedSize = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
1623 FUZ_CHECKTEST(compressedSize==0, "LZ4_compress_HC_continue() compression failed");
1624 FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
1625 DISPLAYLEVEL(5, "compressed %i bytes to %i bytes \n", messageSize, compressedSize);
1626
1627 /* test LZ4_decompress_safe_continue */
1628 assert(dNext < dBufferSize);
1629 assert(dBufferSize - dNext >= maxMessageSize);
1630 result = LZ4_decompress_safe_continue(&decodeStateSafe,
1631 testCompressed, ringBufferSafe + dNext,
1632 compressedSize, dBufferSize - dNext); /* works without knowing messageSize, under assumption that messageSize <= maxMessageSize */
1633 FUZ_CHECKTEST(result!=messageSize, "D.ringBuffer : LZ4_decompress_safe_continue() test failed");
1634 XXH64_update(&xxhNewSafe, ringBufferSafe + dNext, (size_t)messageSize);
1635 { U64 const crcNew = XXH64_digest(&xxhNewSafe);
1636 if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferSafe + dNext);
1637 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_continue() decompression corruption during D.ringBuffer test");
1638 }
1639
1640 /* test LZ4_decompress_fast_continue in its own buffer ringBufferFast */
1641 result = LZ4_decompress_fast_continue(&decodeStateFast, testCompressed, ringBufferFast + dNext, messageSize);
1642 FUZ_CHECKTEST(result!=compressedSize, "D.ringBuffer : LZ4_decompress_fast_continue() test failed");
1643 XXH64_update(&xxhNewFast, ringBufferFast + dNext, (size_t)messageSize);
1644 { U64 const crcNew = XXH64_digest(&xxhNewFast);
1645 if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, ringBufferFast + dNext);
1646 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_fast_continue() decompression corruption during D.ringBuffer test");
1647 }
1648
1649 /* prepare next message */
1650 dNext += messageSize;
1651 assert(messageSize >= 0);
1652 totalMessageSize += (unsigned)messageSize;
1653 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
1654 iNext = (FUZ_rand(&randState) & 65535);
1655 if (dNext + maxMessageSize > dBufferSize) dNext = 0;
1656 }
1657 } /* Ring buffer test : Non synchronized decoder */
1658 }
1659
1660 DISPLAYLEVEL(3, "LZ4_compress_HC_destSize : ");
1661 /* encode congenerical sequence test for HC compressors */
1662 { LZ4_streamHC_t* const sHC = LZ4_createStreamHC();
1663 int const src_buf_size = 3 MB;
1664 int const dst_buf_size = 6 KB;
1665 int const payload = 0;
1666 int const dst_step = 43;
1667 int const dst_min_len = 33 + (FUZ_rand(&randState) % dst_step);
1668 int const dst_max_len = 5000;
1669 int slen, dlen;
1670 char* sbuf1 = (char*)malloc(src_buf_size + 1);
1671 char* sbuf2 = (char*)malloc(src_buf_size + 1);
1672 char* dbuf1 = (char*)malloc(dst_buf_size + 1);
1673 char* dbuf2 = (char*)malloc(dst_buf_size + 1);
1674
1675 assert(sHC != NULL);
1676 assert(dst_buf_size > dst_max_len);
1677 if (!sbuf1 || !sbuf2 || !dbuf1 || !dbuf2) {
1678 EXIT_MSG("not enough memory for FUZ_unitTests (destSize)");
1679 }
1680 for (dlen = dst_min_len; dlen <= dst_max_len; dlen += dst_step) {
1681 int src_len = (dlen - 10)*255 + 24;
1682 if (src_len + 10 >= src_buf_size) break; /* END of check */
1683 for (slen = src_len - 3; slen <= src_len + 3; slen++) {
1684 int srcsz1, srcsz2;
1685 int dsz1, dsz2;
1686 int res1, res2;
1687 char const endchk = (char)0x88;
1688 DISPLAYLEVEL(5, "slen = %i, ", slen);
1689
1690 srcsz1 = slen;
1691 memset(sbuf1, payload, slen);
1692 memset(dbuf1, 0, dlen);
1693 dbuf1[dlen] = endchk;
1694 dsz1 = LZ4_compress_destSize(sbuf1, dbuf1, &srcsz1, dlen);
1695 DISPLAYLEVEL(5, "LZ4_compress_destSize: %i bytes compressed into %i bytes, ", srcsz1, dsz1);
1696 DISPLAYLEVEL(5, "last token : 0x%0X, ", dbuf1[dsz1 - 6]);
1697 DISPLAYLEVEL(5, "last ML extra lenbyte : 0x%0X, \n", dbuf1[dsz1 - 7]);
1698 FUZ_CHECKTEST(dbuf1[dlen] != endchk, "LZ4_compress_destSize() overwrite dst buffer !");
1699 FUZ_CHECKTEST(dsz1 <= 0, "LZ4_compress_destSize() compression failed");
1700 FUZ_CHECKTEST(dsz1 > dlen, "LZ4_compress_destSize() result larger than dst buffer !");
1701 FUZ_CHECKTEST(srcsz1 > slen, "LZ4_compress_destSize() read more than src buffer !");
1702
1703 res1 = LZ4_decompress_safe(dbuf1, sbuf1, dsz1, src_buf_size);
1704 FUZ_CHECKTEST(res1 != srcsz1, "LZ4_compress_destSize() decompression failed!");
1705
1706 srcsz2 = slen;
1707 memset(sbuf2, payload, slen);
1708 memset(dbuf2, 0, dlen);
1709 dbuf2[dlen] = endchk;
1710 LZ4_resetStreamHC(sHC, compressionLevel);
1711 dsz2 = LZ4_compress_HC_destSize(sHC, sbuf2, dbuf2, &srcsz2, dlen, compressionLevel);
1712 DISPLAYLEVEL(5, "LZ4_compress_HC_destSize: %i bytes compressed into %i bytes, ", srcsz2, dsz2);
1713 DISPLAYLEVEL(5, "last token : 0x%0X, ", dbuf2[dsz2 - 6]);
1714 DISPLAYLEVEL(5, "last ML extra lenbyte : 0x%0X, \n", dbuf2[dsz2 - 7]);
1715 FUZ_CHECKTEST(dbuf2[dlen] != endchk, "LZ4_compress_HC_destSize() overwrite dst buffer !");
1716 FUZ_CHECKTEST(dsz2 <= 0, "LZ4_compress_HC_destSize() compression failed");
1717 FUZ_CHECKTEST(dsz2 > dlen, "LZ4_compress_HC_destSize() result larger than dst buffer !");
1718 FUZ_CHECKTEST(srcsz2 > slen, "LZ4_compress_HC_destSize() read more than src buffer !");
1719 FUZ_CHECKTEST(dsz2 != dsz1, "LZ4_compress_HC_destSize() return incorrect result !");
1720 FUZ_CHECKTEST(srcsz2 != srcsz1, "LZ4_compress_HC_destSize() return incorrect src buffer size "
1721 ": srcsz2(%i) != srcsz1(%i)", srcsz2, srcsz1);
1722 FUZ_CHECKTEST(memcmp(dbuf2, dbuf1, (size_t)dsz2), "LZ4_compress_HC_destSize() return incorrect data into dst buffer !");
1723
1724 res2 = LZ4_decompress_safe(dbuf2, sbuf1, dsz2, src_buf_size);
1725 FUZ_CHECKTEST(res2 != srcsz1, "LZ4_compress_HC_destSize() decompression failed!");
1726
1727 FUZ_CHECKTEST(memcmp(sbuf1, sbuf2, (size_t)res2), "LZ4_compress_HC_destSize() decompression corruption!");
1728 }
1729 }
1730 LZ4_freeStreamHC(sHC);
1731 free(sbuf1);
1732 free(sbuf2);
1733 free(dbuf1);
1734 free(dbuf2);
1735 }
1736 DISPLAYLEVEL(3, " OK \n");
1737
1738
1739 /* clean up */
1740 free(testInput);
1741 free(testCompressed);
1742 free(testVerify);
1743
1744 printf("All unit tests completed successfully compressionLevel=%d \n", compressionLevel);
1745 return;
1746 }
1747
1748
1749
1750 /* =======================================
1751 * CLI
1752 * ======================================= */
1753
FUZ_usage(const char * programName)1754 static int FUZ_usage(const char* programName)
1755 {
1756 DISPLAY( "Usage :\n");
1757 DISPLAY( " %s [args]\n", programName);
1758 DISPLAY( "\n");
1759 DISPLAY( "Arguments :\n");
1760 DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS);
1761 DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
1762 DISPLAY( " -s# : Select seed (default:prompt user)\n");
1763 DISPLAY( " -t# : Select starting test number (default:0)\n");
1764 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
1765 DISPLAY( " -v : verbose\n");
1766 DISPLAY( " -p : pause at the end\n");
1767 DISPLAY( " -h : display help and exit\n");
1768 return 0;
1769 }
1770
1771
main(int argc,const char ** argv)1772 int main(int argc, const char** argv)
1773 {
1774 U32 seed = 0;
1775 int seedset = 0;
1776 int argNb;
1777 unsigned nbTests = NB_ATTEMPTS;
1778 unsigned testNb = 0;
1779 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
1780 int use_pause = 0;
1781 const char* programName = argv[0];
1782 U32 duration = 0;
1783
1784 /* Check command line */
1785 for(argNb=1; argNb<argc; argNb++) {
1786 const char* argument = argv[argNb];
1787
1788 if(!argument) continue; // Protection if argument empty
1789
1790 // Decode command (note : aggregated commands are allowed)
1791 if (argument[0]=='-') {
1792 if (!strcmp(argument, "--no-prompt")) { use_pause=0; seedset=1; g_displayLevel=1; continue; }
1793 argument++;
1794
1795 while (*argument!=0) {
1796 switch(*argument)
1797 {
1798 case 'h': /* display help */
1799 return FUZ_usage(programName);
1800
1801 case 'v': /* verbose mode */
1802 g_displayLevel++;
1803 argument++;
1804 break;
1805
1806 case 'p': /* pause at the end */
1807 use_pause=1;
1808 argument++;
1809 break;
1810
1811 case 'i':
1812 argument++;
1813 nbTests = 0; duration = 0;
1814 while ((*argument>='0') && (*argument<='9')) {
1815 nbTests *= 10;
1816 nbTests += (unsigned)(*argument - '0');
1817 argument++;
1818 }
1819 break;
1820
1821 case 'T':
1822 argument++;
1823 nbTests = 0; duration = 0;
1824 for (;;) {
1825 switch(*argument)
1826 {
1827 case 'm': duration *= 60; argument++; continue;
1828 case 's':
1829 case 'n': argument++; continue;
1830 case '0':
1831 case '1':
1832 case '2':
1833 case '3':
1834 case '4':
1835 case '5':
1836 case '6':
1837 case '7':
1838 case '8':
1839 case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue;
1840 }
1841 break;
1842 }
1843 break;
1844
1845 case 's':
1846 argument++;
1847 seed=0; seedset=1;
1848 while ((*argument>='0') && (*argument<='9')) {
1849 seed *= 10;
1850 seed += (U32)(*argument - '0');
1851 argument++;
1852 }
1853 break;
1854
1855 case 't': /* select starting test nb */
1856 argument++;
1857 testNb=0;
1858 while ((*argument>='0') && (*argument<='9')) {
1859 testNb *= 10;
1860 testNb += (unsigned)(*argument - '0');
1861 argument++;
1862 }
1863 break;
1864
1865 case 'P': /* change probability */
1866 argument++;
1867 proba=0;
1868 while ((*argument>='0') && (*argument<='9')) {
1869 proba *= 10;
1870 proba += *argument - '0';
1871 argument++;
1872 }
1873 if (proba<0) proba=0;
1874 if (proba>100) proba=100;
1875 break;
1876 default: ;
1877 }
1878 }
1879 }
1880 }
1881
1882 printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_versionString());
1883
1884 if (!seedset) {
1885 time_t const t = time(NULL);
1886 U32 const h = XXH32(&t, sizeof(t), 1);
1887 seed = h % 10000;
1888 }
1889 printf("Seed = %u\n", seed);
1890
1891 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba);
1892
1893 if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); }
1894
1895 nbTests += (nbTests==0); /* avoid zero */
1896
1897 { int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration);
1898 if (use_pause) {
1899 DISPLAY("press enter ... \n");
1900 (void)getchar();
1901 }
1902 return result;
1903 }
1904 }
1905