1/* 2 frameTest - test tool for lz4frame 3 Copyright (C) Yann Collet 2014-2020 4 5 GPL v2 License 6 7 This program is free software; you can redistribute it and/or modify 8 it under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 2 of the License, or 10 (at your option) any later version. 11 12 This program is distributed in the hope that it will be useful, 13 but WITHOUT ANY WARRANTY; without even the implied warranty of 14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 GNU General Public License for more details. 16 17 You should have received a copy of the GNU General Public License along 18 with this program; if not, write to the Free Software Foundation, Inc., 19 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 20 21 You can contact the author at : 22 - LZ4 homepage : http://www.lz4.org 23 - LZ4 source repository : https://github.com/lz4/lz4 24*/ 25 26/*-************************************ 27* Compiler specific 28**************************************/ 29#ifdef _MSC_VER /* Visual Studio */ 30# pragma warning(disable : 26451) /* disable: Arithmetic overflow */ 31#endif 32 33 34/*-************************************ 35* Includes 36**************************************/ 37#include "util.h" /* U32 */ 38#include <stdlib.h> /* malloc, free */ 39#include <stdio.h> /* fprintf */ 40#include <string.h> /* strcmp */ 41#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */ 42#include <assert.h> 43#include "lz4frame.h" /* included multiple times to test correctness/safety */ 44#include "lz4frame.h" 45#define LZ4F_STATIC_LINKING_ONLY 46#include "lz4frame.h" 47#include "lz4frame.h" 48#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */ 49#include "lz4.h" /* LZ4_VERSION_STRING */ 50#define XXH_STATIC_LINKING_ONLY 51#include "xxhash.h" /* XXH64 */ 52 53 54/* unoptimized version; solves endianness & alignment issues */ 55static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32) 56{ 57 BYTE* const dstPtr = (BYTE*)dstVoidPtr; 58 dstPtr[0] = (BYTE) value32; 59 dstPtr[1] = (BYTE)(value32 >> 8); 60 dstPtr[2] = (BYTE)(value32 >> 16); 61 dstPtr[3] = (BYTE)(value32 >> 24); 62} 63 64 65/*-************************************ 66* Constants 67**************************************/ 68#define KB *(1U<<10) 69#define MB *(1U<<20) 70#define GB *(1U<<30) 71 72static const U32 nbTestsDefault = 256 KB; 73#define FUZ_COMPRESSIBILITY_DEFAULT 50 74static const U32 prime1 = 2654435761U; 75static const U32 prime2 = 2246822519U; 76 77 78/*-************************************ 79* Macros 80**************************************/ 81#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 82#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } 83#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \ 84 if ((FUZ_GetClockSpan(g_clockTime) > refreshRate) || (displayLevel>=4)) \ 85 { g_clockTime = clock(); DISPLAY(__VA_ARGS__); \ 86 if (displayLevel>=4) fflush(stdout); } } 87static const clock_t refreshRate = CLOCKS_PER_SEC / 6; 88static clock_t g_clockTime = 0; 89 90 91/*-*************************************** 92* Local Parameters 93*****************************************/ 94static U32 no_prompt = 0; 95static U32 displayLevel = 2; 96static U32 use_pause = 0; 97 98 99/*-******************************************************* 100* Fuzzer functions 101*********************************************************/ 102#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) 103#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) 104 105typedef struct { 106 int nbAllocs; 107} Test_alloc_state; 108static Test_alloc_state g_testAllocState = { 0 }; 109 110static void* dummy_malloc(void* state, size_t s) 111{ 112 Test_alloc_state* const t = (Test_alloc_state*)state; 113 void* const p = malloc(s); 114 if (p==NULL) return NULL; 115 assert(t != NULL); 116 t->nbAllocs += 1; 117 DISPLAYLEVEL(6, "Allocating %zu bytes at address %p \n", s, p); 118 DISPLAYLEVEL(5, "nb allocated memory segments : %i \n", t->nbAllocs); 119 return p; 120} 121 122static void* dummy_calloc(void* state, size_t s) 123{ 124 Test_alloc_state* const t = (Test_alloc_state*)state; 125 void* const p = calloc(1, s); 126 if (p==NULL) return NULL; 127 assert(t != NULL); 128 t->nbAllocs += 1; 129 DISPLAYLEVEL(6, "Allocating and zeroing %zu bytes at address %p \n", s, p); 130 DISPLAYLEVEL(5, "nb allocated memory segments : %i \n", t->nbAllocs); 131 return p; 132} 133 134static void dummy_free(void* state, void* p) 135{ 136 Test_alloc_state* const t = (Test_alloc_state*)state; 137 if (p==NULL) { 138 DISPLAYLEVEL(5, "free() on NULL \n"); 139 return; 140 } 141 DISPLAYLEVEL(6, "freeing memory at address %p \n", p); 142 free(p); 143 assert(t != NULL); 144 t->nbAllocs -= 1; 145 DISPLAYLEVEL(5, "nb of allocated memory segments after this free : %i \n", t->nbAllocs); 146 assert(t->nbAllocs >= 0); 147} 148 149static const LZ4F_CustomMem lz4f_cmem_test = { 150 dummy_malloc, 151 dummy_calloc, 152 dummy_free, 153 &g_testAllocState 154}; 155 156 157static clock_t FUZ_GetClockSpan(clock_t clockStart) 158{ 159 return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */ 160} 161 162#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) 163unsigned int FUZ_rand(unsigned int* src) 164{ 165 U32 rand32 = *src; 166 rand32 *= prime1; 167 rand32 += prime2; 168 rand32 = FUZ_rotl32(rand32, 13); 169 *src = rand32; 170 return rand32 >> 5; 171} 172 173#define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF) 174#define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15) 175static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed) 176{ 177 BYTE* BBuffer = (BYTE*)buffer; 178 size_t pos = 0; 179 U32 P32 = (U32)(32768 * proba); 180 181 /* First Byte */ 182 BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); 183 184 while (pos < bufferSize) { 185 /* Select : Literal (noise) or copy (within 64K) */ 186 if (FUZ_RAND15BITS < P32) { 187 /* Copy (within 64K) */ 188 size_t const lengthRand = FUZ_RANDLENGTH + 4; 189 size_t const length = MIN(lengthRand, bufferSize - pos); 190 size_t const end = pos + length; 191 size_t const offsetRand = FUZ_RAND15BITS + 1; 192 size_t const offset = MIN(offsetRand, pos); 193 size_t match = pos - offset; 194 while (pos < end) BBuffer[pos++] = BBuffer[match++]; 195 } else { 196 /* Literal (noise) */ 197 size_t const lengthRand = FUZ_RANDLENGTH + 4; 198 size_t const length = MIN(lengthRand, bufferSize - pos); 199 size_t const end = pos + length; 200 while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5); 201 } } 202} 203 204 205static unsigned FUZ_highbit(U32 v32) 206{ 207 unsigned nbBits = 0; 208 if (v32==0) return 0; 209 while (v32) {v32 >>= 1; nbBits ++;} 210 return nbBits; 211} 212 213 214/*-******************************************************* 215* Tests 216*********************************************************/ 217#define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) { fprintf(stderr, "%s \n", LZ4F_getErrorName(v)); goto _output_error; } 218#define CHECK(f) { LZ4F_errorCode_t const CHECK_V(err_ , f); } 219 220int basicTests(U32 seed, double compressibility) 221{ 222#define COMPRESSIBLE_NOISE_LENGTH (2 MB) 223 void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); 224 size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL); 225 void* const compressedBuffer = malloc(cBuffSize); 226 void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); 227 U32 randState = seed; 228 size_t cSize, testSize; 229 LZ4F_decompressionContext_t dCtx = NULL; 230 LZ4F_compressionContext_t cctx = NULL; 231 U64 crcOrig; 232 int basicTests_error = 0; 233 LZ4F_preferences_t prefs; 234 memset(&prefs, 0, sizeof(prefs)); 235 236 if (!CNBuffer || !compressedBuffer || !decodedBuffer) { 237 DISPLAY("allocation error, not enough memory to start fuzzer tests \n"); 238 goto _output_error; 239 } 240 FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); 241 crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); 242 243 /* LZ4F_compressBound() : special case : srcSize == 0 */ 244 DISPLAYLEVEL(3, "LZ4F_compressBound(0) = "); 245 { size_t const cBound = LZ4F_compressBound(0, NULL); 246 if (cBound < 64 KB) goto _output_error; 247 DISPLAYLEVEL(3, " %u \n", (U32)cBound); 248 } 249 250 /* LZ4F_compressBound() : special case : automatic flushing enabled */ 251 DISPLAYLEVEL(3, "LZ4F_compressBound(1 KB, autoFlush=1) = "); 252 { size_t cBound; 253 LZ4F_preferences_t autoFlushPrefs; 254 memset(&autoFlushPrefs, 0, sizeof(autoFlushPrefs)); 255 autoFlushPrefs.autoFlush = 1; 256 cBound = LZ4F_compressBound(1 KB, &autoFlushPrefs); 257 if (cBound > 64 KB) goto _output_error; 258 DISPLAYLEVEL(3, " %u \n", (U32)cBound); 259 } 260 261 /* LZ4F_compressBound() : special case : automatic flushing disabled */ 262 DISPLAYLEVEL(3, "LZ4F_compressBound(1 KB, autoFlush=0) = "); 263 { size_t const cBound = LZ4F_compressBound(1 KB, &prefs); 264 if (cBound < 64 KB) goto _output_error; 265 DISPLAYLEVEL(3, " %u \n", (U32)cBound); 266 } 267 268 /* Special case : null-content frame */ 269 testSize = 0; 270 DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : "); 271 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL)); 272 DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize); 273 274 DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n"); 275 CHECK ( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); 276 277 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n"); 278 assert(cSize >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH); 279 { LZ4F_frameInfo_t frame_info; 280 size_t const fhs = LZ4F_headerSize(compressedBuffer, LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH); 281 size_t avail_in = fhs; 282 CHECK( fhs ); 283 CHECK( LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in) ); 284 if (avail_in != fhs) goto _output_error; /* must consume all, since header size is supposed to be exact */ 285 } 286 287 DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n"); 288 CHECK( LZ4F_freeDecompressionContext(dCtx) ); 289 dCtx = NULL; 290 291 /* test one-pass frame compression */ 292 testSize = COMPRESSIBLE_NOISE_LENGTH; 293 294 DISPLAYLEVEL(3, "LZ4F_compressFrame, using fast level -3 : "); 295 { LZ4F_preferences_t fastCompressPrefs; 296 memset(&fastCompressPrefs, 0, sizeof(fastCompressPrefs)); 297 fastCompressPrefs.compressionLevel = -3; 298 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs)); 299 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); 300 } 301 302 DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : "); 303 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL)); 304 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); 305 306 DISPLAYLEVEL(3, "Decompression test : \n"); 307 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; 308 size_t compressedBufferSize = cSize; 309 310 CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); 311 312 DISPLAYLEVEL(3, "Single Pass decompression : "); 313 CHECK( LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL) ); 314 { U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1); 315 if (crcDest != crcOrig) goto _output_error; } 316 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize); 317 318 DISPLAYLEVEL(3, "Reusing decompression context \n"); 319 { size_t const missingBytes = 4; 320 size_t iSize = compressedBufferSize - missingBytes; 321 const BYTE* cBuff = (const BYTE*) compressedBuffer; 322 BYTE* const ostart = (BYTE*)decodedBuffer; 323 BYTE* op = ostart; 324 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; 325 size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH; 326 DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes); 327 CHECK_V(decResult, LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL)); 328 if (decResult != missingBytes) { 329 DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult); 330 goto _output_error; 331 } 332 DISPLAYLEVEL(3, "indeed, requests %u bytes \n", (unsigned)decResult); 333 cBuff += iSize; 334 iSize = decResult; 335 op += oSize; 336 oSize = (size_t)(oend-op); 337 decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL); 338 if (decResult != 0) goto _output_error; /* should finish now */ 339 op += oSize; 340 if (op>oend) { DISPLAY("decompression write overflow \n"); goto _output_error; } 341 { U64 const crcDest = XXH64(decodedBuffer, (size_t)(op-ostart), 1); 342 if (crcDest != crcOrig) goto _output_error; 343 } } 344 345 { size_t oSize = 0; 346 size_t iSize = 0; 347 LZ4F_frameInfo_t fi; 348 const BYTE* ip = (BYTE*)compressedBuffer; 349 350 DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); 351 CHECK( LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL) ); 352 //DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); 353 DISPLAYLEVEL(3, " OK \n"); 354 355 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on zero-size input : "); 356 { size_t nullSize = 0; 357 size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &nullSize); 358 if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) { 359 DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", 360 LZ4F_getErrorName(fiError)); 361 goto _output_error; 362 } 363 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError)); 364 } 365 366 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on not enough input : "); 367 { size_t inputSize = 6; 368 size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &inputSize); 369 if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) { 370 DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError)); 371 goto _output_error; 372 } 373 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError)); 374 } 375 376 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : "); 377 iSize = LZ4F_headerSize(ip, LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH); 378 CHECK( iSize ); 379 CHECK( LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize) ); 380 DISPLAYLEVEL(3, " correctly decoded \n"); 381 } 382 383 DISPLAYLEVEL(3, "Decode a buggy input : "); 384 assert(COMPRESSIBLE_NOISE_LENGTH > 64); 385 assert(cSize > 48); 386 memcpy(decodedBuffer, (char*)compressedBuffer+16, 32); /* save correct data */ 387 memcpy((char*)compressedBuffer+16, (const char*)decodedBuffer+32, 32); /* insert noise */ 388 { size_t dbSize = COMPRESSIBLE_NOISE_LENGTH; 389 size_t cbSize = cSize; 390 size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &dbSize, 391 compressedBuffer, &cbSize, 392 NULL); 393 if (!LZ4F_isError(decompressError)) goto _output_error; 394 DISPLAYLEVEL(3, "error detected : %s \n", LZ4F_getErrorName(decompressError)); 395 } 396 memcpy((char*)compressedBuffer+16, decodedBuffer, 32); /* restore correct data */ 397 398 DISPLAYLEVEL(3, "Reset decompression context, since it's left in error state \n"); 399 LZ4F_resetDecompressionContext(dCtx); /* always successful */ 400 401 DISPLAYLEVEL(3, "Byte after byte : "); 402 { BYTE* const ostart = (BYTE*)decodedBuffer; 403 BYTE* op = ostart; 404 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; 405 const BYTE* ip = (const BYTE*) compressedBuffer; 406 const BYTE* const iend = ip + cSize; 407 while (ip < iend) { 408 size_t oSize = (size_t)(oend-op); 409 size_t iSize = 1; 410 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); 411 op += oSize; 412 ip += iSize; 413 } 414 { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); 415 if (crcDest != crcOrig) goto _output_error; 416 } 417 DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), (unsigned)COMPRESSIBLE_NOISE_LENGTH); 418 } 419 } 420 421 DISPLAYLEVEL(3, "Using 64 KB block : "); 422 prefs.frameInfo.blockSizeID = LZ4F_max64KB; 423 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; 424 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs)); 425 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 426 427 DISPLAYLEVEL(3, "without checksum : "); 428 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; 429 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs)); 430 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 431 432 DISPLAYLEVEL(3, "Using 256 KB block : "); 433 prefs.frameInfo.blockSizeID = LZ4F_max256KB; 434 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; 435 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs)); 436 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 437 438 DISPLAYLEVEL(3, "Decompression test : \n"); 439 { size_t const decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; 440 unsigned const maxBits = FUZ_highbit((U32)decodedBufferSize); 441 BYTE* const ostart = (BYTE*)decodedBuffer; 442 BYTE* op = ostart; 443 BYTE* const oend = ostart + COMPRESSIBLE_NOISE_LENGTH; 444 const BYTE* ip = (const BYTE*)compressedBuffer; 445 const BYTE* const iend = (const BYTE*)compressedBuffer + cSize; 446 447 DISPLAYLEVEL(3, "random segment sizes : "); 448 while (ip < iend) { 449 unsigned const nbBits = FUZ_rand(&randState) % maxBits; 450 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; 451 size_t oSize = (size_t)(oend-op); 452 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip); 453 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); 454 op += oSize; 455 ip += iSize; 456 } 457 { size_t const decodedSize = (size_t)(op - ostart); 458 U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1); 459 if (crcDest != crcOrig) goto _output_error; 460 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); 461 } 462 463 CHECK( LZ4F_freeDecompressionContext(dCtx) ); 464 dCtx = NULL; 465 } 466 467 DISPLAYLEVEL(3, "without checksum : "); 468 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; 469 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); 470 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 471 472 DISPLAYLEVEL(3, "Using 1 MB block : "); 473 prefs.frameInfo.blockSizeID = LZ4F_max1MB; 474 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; 475 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); 476 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 477 478 DISPLAYLEVEL(3, "without frame checksum : "); 479 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; 480 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); 481 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 482 483 DISPLAYLEVEL(3, "Using 4 MB block : "); 484 prefs.frameInfo.blockSizeID = LZ4F_max4MB; 485 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; 486 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); 487 DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity) 488 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) ); 489 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); 490 } 491 492 DISPLAYLEVEL(3, "without frame checksum : "); 493 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; 494 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs); 495 DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity) 496 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) ); 497 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize); 498 } 499 500 DISPLAYLEVEL(3, "LZ4F_compressFrame with block checksum : "); 501 memset(&prefs, 0, sizeof(prefs)); 502 prefs.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; 503 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) ); 504 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); 505 506 DISPLAYLEVEL(3, "Decompress with block checksum : "); 507 { size_t iSize = cSize; 508 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; 509 LZ4F_decompressionContext_t dctx; 510 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); 511 CHECK( LZ4F_decompress(dctx, decodedBuffer, &decodedSize, compressedBuffer, &iSize, NULL) ); 512 if (decodedSize != testSize) goto _output_error; 513 if (iSize != cSize) goto _output_error; 514 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1); 515 U64 const crcSrc = XXH64(CNBuffer, testSize, 1); 516 if (crcDest != crcSrc) goto _output_error; 517 } 518 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); 519 520 CHECK( LZ4F_freeDecompressionContext(dctx) ); 521 } 522 523 /* frame content size tests */ 524 { size_t cErr; 525 BYTE* const ostart = (BYTE*)compressedBuffer; 526 BYTE* op = ostart; 527 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); 528 529 DISPLAYLEVEL(3, "compress without frameSize : "); 530 memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo)); 531 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); 532 op += cErr; 533 CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL)); 534 op += cErr; 535 CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) ); 536 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); 537 538 DISPLAYLEVEL(3, "compress with frameSize : "); 539 prefs.frameInfo.contentSize = testSize; 540 op = ostart; 541 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); 542 op += cErr; 543 CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL)); 544 op += cErr; 545 CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) ); 546 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart)); 547 548 DISPLAYLEVEL(3, "compress with wrong frameSize : "); 549 prefs.frameInfo.contentSize = testSize+1; 550 op = ostart; 551 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); 552 op += cErr; 553 CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL)); 554 op += cErr; 555 cErr = LZ4F_compressEnd(cctx, op, testSize, NULL); 556 if (!LZ4F_isError(cErr)) goto _output_error; 557 DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(cErr)); 558 559 CHECK( LZ4F_freeCompressionContext(cctx) ); 560 cctx = NULL; 561 } 562 563 /* dictID tests */ 564 { size_t cErr; 565 U32 const dictID = 0x99; 566 /* test advanced variant with custom allocator functions */ 567 cctx = LZ4F_createCompressionContext_advanced(lz4f_cmem_test, LZ4F_VERSION); 568 if (cctx==NULL) goto _output_error; 569 570 DISPLAYLEVEL(3, "insert a dictID : "); 571 memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo)); 572 prefs.frameInfo.dictID = dictID; 573 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs)); 574 DISPLAYLEVEL(3, "created frame header of size %i bytes \n", (int)cErr); 575 576 DISPLAYLEVEL(3, "read a dictID : "); 577 CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); 578 memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo)); 579 CHECK( LZ4F_getFrameInfo(dCtx, &prefs.frameInfo, compressedBuffer, &cErr) ); 580 if (prefs.frameInfo.dictID != dictID) goto _output_error; 581 DISPLAYLEVEL(3, "%u \n", (U32)prefs.frameInfo.dictID); 582 583 CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL; 584 CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL; 585 } 586 587 /* Dictionary compression test */ 588 { size_t const dictSize = 7 KB; /* small enough for LZ4_MEMORY_USAGE == 10 */ 589 size_t const srcSize = 65 KB; /* must be > 64 KB to avoid short-size optimizations */ 590 size_t const dstCapacity = LZ4F_compressFrameBound(srcSize, NULL); 591 size_t cSizeNoDict, cSizeWithDict; 592 LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize); 593 if (cdict == NULL) goto _output_error; 594 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); 595 596 DISPLAYLEVEL(3, "Testing LZ4F_createCDict_advanced : "); 597 { LZ4F_CDict* const cda = LZ4F_createCDict_advanced(lz4f_cmem_test, CNBuffer, dictSize); 598 if (cda == NULL) goto _output_error; 599 LZ4F_freeCDict(cda); 600 } 601 DISPLAYLEVEL(3, "OK \n"); 602 603 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : "); 604 CHECK_V(cSizeNoDict, 605 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, 606 CNBuffer, srcSize, 607 NULL, NULL) ); 608 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict); 609 610 CHECK( LZ4F_freeCompressionContext(cctx) ); 611 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) ); 612 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : "); 613 CHECK_V(cSizeWithDict, 614 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, 615 CNBuffer, srcSize, 616 cdict, NULL) ); 617 DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", 618 (unsigned)srcSize, (unsigned)cSizeWithDict); 619 if (cSizeWithDict > cSizeNoDict) { 620 DISPLAYLEVEL(3, "cSizeWithDict (%zu) should have been more compact than cSizeNoDict(%zu) \n", cSizeWithDict, cSizeNoDict); 621 goto _output_error; /* must be more efficient */ 622 } 623 crcOrig = XXH64(CNBuffer, srcSize, 0); 624 625 DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : "); 626 { LZ4F_dctx* dctx; 627 size_t decodedSize = srcSize; 628 size_t compressedSize = cSizeWithDict; 629 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); 630 CHECK( LZ4F_decompress_usingDict(dctx, 631 decodedBuffer, &decodedSize, 632 compressedBuffer, &compressedSize, 633 CNBuffer, dictSize, 634 NULL) ); 635 if (compressedSize != cSizeWithDict) goto _output_error; 636 if (decodedSize != srcSize) goto _output_error; 637 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); 638 if (crcDest != crcOrig) goto _output_error; } 639 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); 640 CHECK( LZ4F_freeDecompressionContext(dctx) ); 641 } 642 643 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, negative level : "); 644 { size_t cSizeLevelMax; 645 LZ4F_preferences_t cParams; 646 memset(&cParams, 0, sizeof(cParams)); 647 cParams.compressionLevel = -3; 648 CHECK_V(cSizeLevelMax, 649 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, 650 CNBuffer, dictSize, 651 cdict, &cParams) ); 652 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax); 653 } 654 655 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, level max : "); 656 { size_t cSizeLevelMax; 657 LZ4F_preferences_t cParams; 658 memset(&cParams, 0, sizeof(cParams)); 659 cParams.compressionLevel = LZ4F_compressionLevel_max(); 660 CHECK_V(cSizeLevelMax, 661 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity, 662 CNBuffer, dictSize, 663 cdict, &cParams) ); 664 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax); 665 } 666 667 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple linked blocks : "); 668 { size_t cSizeContiguous; 669 size_t const inSize = dictSize * 3; 670 size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL); 671 LZ4F_preferences_t cParams; 672 memset(&cParams, 0, sizeof(cParams)); 673 cParams.frameInfo.blockMode = LZ4F_blockLinked; 674 cParams.frameInfo.blockSizeID = LZ4F_max64KB; 675 CHECK_V(cSizeContiguous, 676 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity, 677 CNBuffer, inSize, 678 cdict, &cParams) ); 679 DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", 680 (unsigned)inSize, (unsigned)cSizeContiguous); 681 682 DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple linked blocks : "); 683 { LZ4F_dctx* dctx; 684 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; 685 size_t compressedSize = cSizeContiguous; 686 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); 687 CHECK( LZ4F_decompress_usingDict(dctx, 688 decodedBuffer, &decodedSize, 689 compressedBuffer, &compressedSize, 690 CNBuffer, dictSize, 691 NULL) ); 692 if (compressedSize != cSizeContiguous) goto _output_error; 693 if (decodedSize != inSize) goto _output_error; 694 crcOrig = XXH64(CNBuffer, inSize, 0); 695 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); 696 if (crcDest != crcOrig) goto _output_error; } 697 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); 698 CHECK( LZ4F_freeDecompressionContext(dctx) ); 699 } 700 } 701 702 703 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple independent blocks : "); 704 { size_t cSizeIndep; 705 size_t const inSize = dictSize * 3; 706 size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL); 707 LZ4F_preferences_t cParams; 708 memset(&cParams, 0, sizeof(cParams)); 709 cParams.frameInfo.blockMode = LZ4F_blockIndependent; 710 cParams.frameInfo.blockSizeID = LZ4F_max64KB; 711 CHECK_V(cSizeIndep, 712 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity, 713 CNBuffer, inSize, 714 cdict, &cParams) ); 715 DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n", 716 (unsigned)inSize, (unsigned)cSizeIndep); 717 718 DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple independent blocks : "); 719 { LZ4F_dctx* dctx; 720 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH; 721 size_t compressedSize = cSizeIndep; 722 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) ); 723 CHECK( LZ4F_decompress_usingDict(dctx, 724 decodedBuffer, &decodedSize, 725 compressedBuffer, &compressedSize, 726 CNBuffer, dictSize, 727 NULL) ); 728 if (compressedSize != cSizeIndep) goto _output_error; 729 if (decodedSize != inSize) goto _output_error; 730 crcOrig = XXH64(CNBuffer, inSize, 0); 731 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0); 732 if (crcDest != crcOrig) goto _output_error; } 733 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize); 734 CHECK( LZ4F_freeDecompressionContext(dctx) ); 735 } 736 } 737 738 LZ4F_freeCDict(cdict); 739 CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL; 740 } 741 742 DISPLAYLEVEL(3, "getBlockSize test: \n"); 743 { size_t result; 744 unsigned blockSizeID; 745 for (blockSizeID = 4; blockSizeID < 8; ++blockSizeID) { 746 result = LZ4F_getBlockSize((LZ4F_blockSizeID_t)blockSizeID); 747 CHECK(result); 748 DISPLAYLEVEL(3, "Returned block size of %u bytes for blockID %u \n", 749 (unsigned)result, blockSizeID); 750 } 751 752 /* Test an invalid input that's too large */ 753 result = LZ4F_getBlockSize((LZ4F_blockSizeID_t)8); 754 if(!LZ4F_isError(result) || 755 LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid) 756 goto _output_error; 757 758 /* Test an invalid input that's too small */ 759 result = LZ4F_getBlockSize((LZ4F_blockSizeID_t)3); 760 if(!LZ4F_isError(result) || 761 LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid) 762 goto _output_error; 763 } 764 765 766 DISPLAYLEVEL(3, "Skippable frame test : \n"); 767 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH; 768 unsigned maxBits = FUZ_highbit((U32)decodedBufferSize); 769 BYTE* op = (BYTE*)decodedBuffer; 770 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH; 771 BYTE* ip = (BYTE*)compressedBuffer; 772 BYTE* iend = (BYTE*)compressedBuffer + cSize + 8; 773 774 CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) ); 775 776 /* generate skippable frame */ 777 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START); 778 FUZ_writeLE32(ip+4, (U32)cSize); 779 780 DISPLAYLEVEL(3, "random segment sizes : \n"); 781 while (ip < iend) { 782 unsigned nbBits = FUZ_rand(&randState) % maxBits; 783 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; 784 size_t oSize = (size_t)(oend-op); 785 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip); 786 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); 787 op += oSize; 788 ip += iSize; 789 } 790 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize); 791 792 /* generate zero-size skippable frame */ 793 DISPLAYLEVEL(3, "zero-size skippable frame\n"); 794 ip = (BYTE*)compressedBuffer; 795 op = (BYTE*)decodedBuffer; 796 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1); 797 FUZ_writeLE32(ip+4, 0); 798 iend = ip+8; 799 800 while (ip < iend) { 801 unsigned const nbBits = FUZ_rand(&randState) % maxBits; 802 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1; 803 size_t oSize = (size_t)(oend-op); 804 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip); 805 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); 806 op += oSize; 807 ip += iSize; 808 } 809 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); 810 811 DISPLAYLEVEL(3, "Skippable frame header complete in first call \n"); 812 ip = (BYTE*)compressedBuffer; 813 op = (BYTE*)decodedBuffer; 814 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2); 815 FUZ_writeLE32(ip+4, 10); 816 iend = ip+18; 817 while (ip < iend) { 818 size_t iSize = 10; 819 size_t oSize = 10; 820 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip); 821 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) ); 822 op += oSize; 823 ip += iSize; 824 } 825 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); 826 } 827 828 DISPLAY("Basic tests completed \n"); 829_end: 830 free(CNBuffer); 831 free(compressedBuffer); 832 free(decodedBuffer); 833 LZ4F_freeDecompressionContext(dCtx); dCtx = NULL; 834 LZ4F_freeCompressionContext(cctx); cctx = NULL; 835 return basicTests_error; 836 837_output_error: 838 basicTests_error = 1; 839 DISPLAY("Error detected ! \n"); 840 goto _end; 841} 842 843 844typedef enum { o_contiguous, o_noncontiguous, o_overwrite } o_scenario_e; 845 846static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, o_scenario_e o_scenario) 847{ 848 if (displayLevel >= 2) { 849 size_t p=0; 850 const BYTE* b1=(const BYTE*)buff1; 851 const BYTE* b2=(const BYTE*)buff2; 852 DISPLAY("locateBuffDiff: looking for error position \n"); 853 if (o_scenario != o_contiguous) { 854 DISPLAY("mode %i: non-contiguous output (%u bytes), cannot search \n", 855 (int)o_scenario, (unsigned)size); 856 return; 857 } 858 while (p < size && b1[p]==b2[p]) p++; 859 if (p != size) { 860 DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]); 861 } 862 } 863} 864 865# define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ 866 DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); exit(1); } 867# undef CHECK 868# define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } } 869 870 871size_t test_lz4f_decompression_wBuffers( 872 const void* cSrc, size_t cSize, 873 void* dst, size_t dstCapacity, o_scenario_e o_scenario, 874 const void* srcRef, size_t decompressedSize, 875 U64 crcOrig, 876 U32* const randState, 877 LZ4F_dctx* const dCtx, 878 U32 seed, U32 testNb, 879 int findErrorPos) 880{ 881 const BYTE* ip = (const BYTE*)cSrc; 882 const BYTE* const iend = ip + cSize; 883 884 BYTE* op = (BYTE*)dst; 885 BYTE* const oend = op + dstCapacity; 886 887 unsigned const suggestedBits = FUZ_highbit((U32)cSize); 888 unsigned const maxBits = MAX(3, suggestedBits); 889 size_t totalOut = 0; 890 size_t moreToFlush = 0; 891 XXH64_state_t xxh64; 892 XXH64_reset(&xxh64, 1); 893 assert(ip < iend); 894 while (ip < iend) { 895 unsigned const nbBitsI = (FUZ_rand(randState) % (maxBits-1)) + 1; 896 unsigned const nbBitsO = (FUZ_rand(randState) % (maxBits)) + 1; 897 size_t const iSizeCand = (FUZ_rand(randState) & ((1<<nbBitsI)-1)) + 1; 898 size_t const iSizeMax = MIN(iSizeCand, (size_t)(iend-ip)); 899 size_t iSize = iSizeMax; 900 size_t const oSizeCand = (FUZ_rand(randState) & ((1<<nbBitsO)-1)) + 2; 901 size_t const oSizeMax = MIN(oSizeCand, (size_t)(oend-op)); 902 int const sentinelTest = (op + oSizeMax < oend); 903 size_t oSize = oSizeMax; 904 BYTE const mark = (BYTE)(FUZ_rand(randState) & 255); 905 LZ4F_decompressOptions_t dOptions; 906 memset(&dOptions, 0, sizeof(dOptions)); 907 dOptions.stableDst = FUZ_rand(randState) & 1; 908 if (o_scenario == o_overwrite) dOptions.stableDst = 0; /* overwrite mode */ 909 dOptions.skipChecksums = FUZ_rand(randState) & 127; 910 if (sentinelTest) op[oSizeMax] = mark; 911 912 DISPLAYLEVEL(7, "dstCapacity=%u, presentedInput=%u \n", (unsigned)oSize, (unsigned)iSize); 913 914 /* read data from byte-exact buffer to catch out-of-bound reads */ 915 { void* const iBuffer = malloc(iSizeMax); 916 void* const tmpop = (FUZ_rand(randState) & (oSize == 0)) ? NULL : op; 917 const void* const tmpip = (FUZ_rand(randState) & (iSize == 0)) ? NULL : iBuffer; 918 assert(iBuffer != NULL); 919 memcpy(iBuffer, ip, iSizeMax); 920 moreToFlush = LZ4F_decompress(dCtx, tmpop, &oSize, tmpip, &iSize, &dOptions); 921 free(iBuffer); 922 } 923 DISPLAYLEVEL(7, "oSize=%u, readSize=%u \n", (unsigned)oSize, (unsigned)iSize); 924 925 if (sentinelTest) { 926 CHECK(op[oSizeMax] != mark, "op[oSizeMax] = %02X != %02X : " 927 "Decompression overwrites beyond assigned dst size", 928 op[oSizeMax], mark); 929 } 930 if (LZ4F_getErrorCode(moreToFlush) == LZ4F_ERROR_contentChecksum_invalid) { 931 if (findErrorPos) DISPLAYLEVEL(2, "checksum error detected \n"); 932 if (findErrorPos) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario); 933 } 934 if (LZ4F_isError(moreToFlush)) return moreToFlush; 935 936 XXH64_update(&xxh64, op, oSize); 937 totalOut += oSize; 938 op += oSize; 939 ip += iSize; 940 if (o_scenario == o_noncontiguous) { 941 if (op == oend) return LZ4F_ERROR_GENERIC; /* can theoretically happen with bogus data */ 942 op++; /* create a gap between consecutive output */ 943 } 944 if (o_scenario==o_overwrite) op = (BYTE*)dst; /* overwrite destination */ 945 if ( (op == oend) /* no more room for output; can happen with bogus input */ 946 && (iSize == 0)) /* no input consumed */ 947 break; 948 } 949 if (moreToFlush != 0) return LZ4F_ERROR_decompressionFailed; 950 if (totalOut) { /* otherwise, it's a skippable frame */ 951 U64 const crcDecoded = XXH64_digest(&xxh64); 952 if (crcDecoded != crcOrig) { 953 if (findErrorPos) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario); 954 return LZ4F_ERROR_contentChecksum_invalid; 955 } } 956 return 0; 957} 958 959 960size_t test_lz4f_decompression(const void* cSrc, size_t cSize, 961 const void* srcRef, size_t decompressedSize, 962 U64 crcOrig, 963 U32* const randState, 964 LZ4F_dctx* const dCtx, 965 U32 seed, U32 testNb, 966 int findErrorPos) 967{ 968 o_scenario_e const o_scenario = (o_scenario_e)(FUZ_rand(randState) % 3); /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */ 969 /* tighten dst buffer conditions */ 970 size_t const dstCapacity = (o_scenario == o_noncontiguous) ? 971 (decompressedSize * 2) + 128 : 972 decompressedSize; 973 size_t result; 974 void* const dstBuffer = malloc(dstCapacity); 975 assert(dstBuffer != NULL); 976 977 result = test_lz4f_decompression_wBuffers(cSrc, cSize, 978 dstBuffer, dstCapacity, o_scenario, 979 srcRef, decompressedSize, 980 crcOrig, 981 randState, 982 dCtx, 983 seed, testNb, findErrorPos); 984 985 free(dstBuffer); 986 return result; 987} 988 989 990int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s) 991{ 992 unsigned testNb = 0; 993 size_t const CNBufferLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ 994 void* CNBuffer = NULL; 995 size_t const compressedBufferSize = LZ4F_compressFrameBound(CNBufferLength, NULL) + 4 MB; /* needs some margin */ 996 void* compressedBuffer = NULL; 997 void* decodedBuffer = NULL; 998 U32 coreRand = seed; 999 LZ4F_decompressionContext_t dCtx = NULL; 1000 LZ4F_decompressionContext_t dCtxNoise = NULL; 1001 LZ4F_compressionContext_t cCtx = NULL; 1002 clock_t const startClock = clock(); 1003 clock_t const clockDuration = duration_s * CLOCKS_PER_SEC; 1004 1005 /* Create states & buffers */ 1006 { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 1007 CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } 1008 { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtxNoise, LZ4F_VERSION); 1009 CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } 1010 { size_t const creationStatus = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION); 1011 CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); } 1012 CNBuffer = malloc(CNBufferLength); 1013 CHECK(CNBuffer==NULL, "CNBuffer Allocation failed"); 1014 compressedBuffer = malloc(compressedBufferSize); 1015 CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed"); 1016 decodedBuffer = calloc(1, CNBufferLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */ 1017 CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed"); 1018 FUZ_fillCompressibleNoiseBuffer(CNBuffer, CNBufferLength, compressibility, &coreRand); 1019 1020 /* jump to requested testNb */ 1021 for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); /* sync randomizer */ 1022 1023 /* main fuzzer test loop */ 1024 for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) { 1025 U32 randState = coreRand ^ prime1; 1026 unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(CNBufferLength-1)) - 1)) + 1; 1027 size_t const srcSize = (FUZ_rand(&randState) & ((1<<srcBits)-1)) + 1; 1028 size_t const srcStartId = FUZ_rand(&randState) % (CNBufferLength - srcSize); 1029 const BYTE* const srcStart = (const BYTE*)CNBuffer + srcStartId; 1030 unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1; 1031 U64 const crcOrig = XXH64(srcStart, srcSize, 1); 1032 LZ4F_preferences_t prefs; 1033 const LZ4F_preferences_t* prefsPtr = &prefs; 1034 size_t cSize; 1035 1036 (void)FUZ_rand(&coreRand); /* update seed */ 1037 memset(&prefs, 0, sizeof(prefs)); 1038 prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1); 1039 prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3)); 1040 prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)(FUZ_rand(&randState) & 1); 1041 prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1); 1042 prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0; 1043 prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2; 1044 prefs.compressionLevel = -5 + (int)(FUZ_rand(&randState) % 11); 1045 if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL; 1046 1047 DISPLAYUPDATE(2, "\r%5u ", testNb); 1048 1049 if ((FUZ_rand(&randState) & 0xFFF) == 0) { 1050 /* create a skippable frame (rare case) */ 1051 BYTE* op = (BYTE*)compressedBuffer; 1052 FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15)); 1053 FUZ_writeLE32(op+4, (U32)srcSize); 1054 cSize = srcSize+8; 1055 1056 } else if ((FUZ_rand(&randState) & 0xF) == 2) { /* single pass compression (simple) */ 1057 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), srcStart, srcSize, prefsPtr); 1058 CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize)); 1059 1060 } else { /* multi-segments compression */ 1061 const BYTE* ip = srcStart; 1062 const BYTE* const iend = srcStart + srcSize; 1063 BYTE* op = (BYTE*)compressedBuffer; 1064 BYTE* const oend = op + (neverFlush ? LZ4F_compressFrameBound(srcSize, prefsPtr) : compressedBufferSize); /* when flushes are possible, can't guarantee a max compressed size */ 1065 unsigned const maxBits = FUZ_highbit((U32)srcSize); 1066 LZ4F_compressOptions_t cOptions; 1067 memset(&cOptions, 0, sizeof(cOptions)); 1068 { size_t const fhSize = LZ4F_compressBegin(cCtx, op, (size_t)(oend-op), prefsPtr); 1069 CHECK(LZ4F_isError(fhSize), "Compression header failed (error %i)", 1070 (int)fhSize); 1071 op += fhSize; 1072 } 1073 while (ip < iend) { 1074 unsigned const nbBitsSeg = FUZ_rand(&randState) % maxBits; 1075 size_t const sampleMax = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1; 1076 size_t iSize = MIN(sampleMax, (size_t)(iend-ip)); 1077 size_t const oSize = LZ4F_compressBound(iSize, prefsPtr); 1078 cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1); 1079 DISPLAYLEVEL(6, "Sending %u bytes to compress (stableSrc:%u) \n", 1080 (unsigned)iSize, cOptions.stableSrc); 1081 1082#if 1 1083 /* insert uncompressed segment */ 1084 if ( (iSize>0) 1085 && !neverFlush /* do not mess with compressBound when neverFlush is set */ 1086 && prefsPtr != NULL /* prefs are set */ 1087 && prefs.frameInfo.blockMode == LZ4F_blockIndependent /* uncompressedUpdate is only valid with blockMode==independent */ 1088 && (FUZ_rand(&randState) & 15) == 1 ) { 1089 size_t const uSize = FUZ_rand(&randState) % iSize; 1090 size_t const flushedSize = LZ4F_uncompressedUpdate(cCtx, op, (size_t)(oend-op), ip, uSize, &cOptions); 1091 CHECK(LZ4F_isError(flushedSize), "Insert uncompressed data failed (error %i : %s)", 1092 (int)flushedSize, LZ4F_getErrorName(flushedSize)); 1093 op += flushedSize; 1094 ip += uSize; 1095 iSize -= uSize; 1096 } 1097#endif 1098 1099 { size_t const flushedSize = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions); 1100 CHECK(LZ4F_isError(flushedSize), "Compression failed (error %i : %s)", 1101 (int)flushedSize, LZ4F_getErrorName(flushedSize)); 1102 op += flushedSize; 1103 ip += iSize; 1104 } 1105 1106 { unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1); 1107 if (forceFlush) { 1108 size_t const flushSize = LZ4F_flush(cCtx, op, (size_t)(oend-op), &cOptions); 1109 DISPLAYLEVEL(6,"flushing %u bytes \n", (unsigned)flushSize); 1110 CHECK(LZ4F_isError(flushSize), "Compression failed (error %i)", (int)flushSize); 1111 op += flushSize; 1112 if ((FUZ_rand(&randState) % 1024) == 3) { 1113 /* add an empty block (requires uncompressed flag) */ 1114 op[0] = op[1] = op[2] = 0; 1115 op[3] = 0x80; /* 0x80000000U in little-endian format */ 1116 op += 4; 1117 if ((prefsPtr!= NULL) && prefsPtr->frameInfo.blockChecksumFlag) { 1118 /* add block checksum (even for empty blocks) */ 1119 FUZ_writeLE32(op, XXH32(op, 0, 0)); 1120 op += 4; 1121 } } } } 1122 } /* while (ip<iend) */ 1123 CHECK(op>=oend, "LZ4F_compressFrameBound overflow"); 1124 { size_t const dstEndSafeSize = LZ4F_compressBound(0, prefsPtr); 1125 int const tooSmallDstEnd = ((FUZ_rand(&randState) & 31) == 3); 1126 size_t const dstEndTooSmallSize = (FUZ_rand(&randState) % dstEndSafeSize) + 1; 1127 size_t const dstEndSize = tooSmallDstEnd ? dstEndTooSmallSize : dstEndSafeSize; 1128 BYTE const canaryByte = (BYTE)(FUZ_rand(&randState) & 255); 1129 size_t flushedSize; 1130 DISPLAYLEVEL(7,"canaryByte at pos %u / %u \n", 1131 (unsigned)((size_t)(op - (BYTE*)compressedBuffer) + dstEndSize), 1132 (unsigned)compressedBufferSize); 1133 assert(op + dstEndSize < (BYTE*)compressedBuffer + compressedBufferSize); 1134 op[dstEndSize] = canaryByte; 1135 flushedSize = LZ4F_compressEnd(cCtx, op, dstEndSize, &cOptions); 1136 CHECK(op[dstEndSize] != canaryByte, "LZ4F_compressEnd writes beyond dstCapacity !"); 1137 if (LZ4F_isError(flushedSize)) { 1138 if (tooSmallDstEnd) /* failure is allowed */ continue; 1139 CHECK(!tooSmallDstEnd, "Compression completion failed (error %i : %s)", 1140 (int)flushedSize, LZ4F_getErrorName(flushedSize)); 1141 } 1142 op += flushedSize; 1143 } 1144 cSize = (size_t)(op - (BYTE*)compressedBuffer); 1145 DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize); 1146 } 1147 1148 1149 /* multi-segments decompression */ 1150 DISPLAYLEVEL(6, "normal decompression \n"); 1151 { size_t result = test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtx, seed, testNb, 1 /*findError*/ ); 1152 CHECK (LZ4F_isError(result), "multi-segment decompression failed (error %i => %s)", 1153 (int)result, LZ4F_getErrorName(result)); 1154 } 1155 1156#if 1 1157 /* insert noise into src */ 1158 { U32 const maxNbBits = FUZ_highbit((U32)cSize); 1159 size_t pos = 0; 1160 for (;;) { 1161 /* keep some original src */ 1162 { U32 const nbBits = FUZ_rand(&randState) % maxNbBits; 1163 size_t const mask = (1<<nbBits) - 1; 1164 size_t const skipLength = FUZ_rand(&randState) & mask; 1165 pos += skipLength; 1166 } 1167 if (pos >= cSize) break; 1168 /* add noise */ 1169 { U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits; 1170 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; 1171 size_t const mask = (1<<nbBits) - 1; 1172 size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1; 1173 size_t const noiseLength = MIN(rNoiseLength, cSize-pos); 1174 size_t const noiseStart = FUZ_rand(&randState) % (CNBufferLength - noiseLength); 1175 memcpy((BYTE*)compressedBuffer + pos, (const char*)CNBuffer + noiseStart, noiseLength); 1176 pos += noiseLength; 1177 } } } 1178 1179 /* test decompression on noisy src */ 1180 DISPLAYLEVEL(6, "noisy decompression \n"); 1181 test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtxNoise, seed, testNb, 0 /*don't search error Pos*/ ); 1182 /* note : we don't analyze result here : it probably failed, which is expected. 1183 * The sole purpose is to catch potential out-of-bound reads and writes. */ 1184 LZ4F_resetDecompressionContext(dCtxNoise); /* context must be reset after an error */ 1185#endif 1186 1187} /* for ( ; (testNb < nbTests) ; ) */ 1188 1189 DISPLAYLEVEL(2, "\rAll tests completed \n"); 1190 1191 LZ4F_freeDecompressionContext(dCtx); 1192 LZ4F_freeDecompressionContext(dCtxNoise); 1193 LZ4F_freeCompressionContext(cCtx); 1194 free(CNBuffer); 1195 free(compressedBuffer); 1196 free(decodedBuffer); 1197 1198 if (use_pause) { 1199 DISPLAY("press enter to finish \n"); 1200 (void)getchar(); 1201 } 1202 return 0; 1203} 1204 1205 1206int FUZ_usage(const char* programName) 1207{ 1208 DISPLAY( "Usage :\n"); 1209 DISPLAY( " %s [args]\n", programName); 1210 DISPLAY( "\n"); 1211 DISPLAY( "Arguments :\n"); 1212 DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); 1213 DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); 1214 DISPLAY( " -s# : Select seed (default:prompt user)\n"); 1215 DISPLAY( " -t# : Select starting test number (default:0)\n"); 1216 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); 1217 DISPLAY( " -v : verbose\n"); 1218 DISPLAY( " -h : display help and exit\n"); 1219 return 0; 1220} 1221 1222 1223int main(int argc, const char** argv) 1224{ 1225 U32 seed=0; 1226 int seedset=0; 1227 int argNb; 1228 unsigned nbTests = nbTestsDefault; 1229 unsigned testNb = 0; 1230 int proba = FUZ_COMPRESSIBILITY_DEFAULT; 1231 int result=0; 1232 U32 duration=0; 1233 const char* const programName = argv[0]; 1234 1235 /* Check command line */ 1236 for (argNb=1; argNb<argc; argNb++) { 1237 const char* argument = argv[argNb]; 1238 1239 if(!argument) continue; /* Protection if argument empty */ 1240 1241 /* Decode command (note : aggregated short commands are allowed) */ 1242 if (argument[0]=='-') { 1243 if (!strcmp(argument, "--no-prompt")) { 1244 no_prompt=1; 1245 seedset=1; 1246 displayLevel=1; 1247 continue; 1248 } 1249 argument++; 1250 1251 while (*argument!=0) { 1252 switch(*argument) 1253 { 1254 case 'h': 1255 return FUZ_usage(programName); 1256 case 'v': 1257 argument++; 1258 displayLevel++; 1259 break; 1260 case 'q': 1261 argument++; 1262 displayLevel--; 1263 break; 1264 case 'p': /* pause at the end */ 1265 argument++; 1266 use_pause = 1; 1267 break; 1268 1269 case 'i': 1270 argument++; 1271 nbTests=0; duration=0; 1272 while ((*argument>='0') && (*argument<='9')) { 1273 nbTests *= 10; 1274 nbTests += (unsigned)(*argument - '0'); 1275 argument++; 1276 } 1277 break; 1278 1279 case 'T': 1280 argument++; 1281 nbTests = 0; duration = 0; 1282 for (;;) { 1283 switch(*argument) 1284 { 1285 case 'm': duration *= 60; argument++; continue; 1286 case 's': 1287 case 'n': argument++; continue; 1288 case '0': 1289 case '1': 1290 case '2': 1291 case '3': 1292 case '4': 1293 case '5': 1294 case '6': 1295 case '7': 1296 case '8': 1297 case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue; 1298 } 1299 break; 1300 } 1301 break; 1302 1303 case 's': 1304 argument++; 1305 seed=0; 1306 seedset=1; 1307 while ((*argument>='0') && (*argument<='9')) { 1308 seed *= 10; 1309 seed += (U32)(*argument - '0'); 1310 argument++; 1311 } 1312 break; 1313 case 't': 1314 argument++; 1315 testNb=0; 1316 while ((*argument>='0') && (*argument<='9')) { 1317 testNb *= 10; 1318 testNb += (unsigned)(*argument - '0'); 1319 argument++; 1320 } 1321 break; 1322 case 'P': /* compressibility % */ 1323 argument++; 1324 proba=0; 1325 while ((*argument>='0') && (*argument<='9')) { 1326 proba *= 10; 1327 proba += *argument - '0'; 1328 argument++; 1329 } 1330 if (proba<0) proba=0; 1331 if (proba>100) proba=100; 1332 break; 1333 default: 1334 ; 1335 return FUZ_usage(programName); 1336 } 1337 } 1338 } 1339 } 1340 1341 /* Get Seed */ 1342 DISPLAY("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING); 1343 1344 if (!seedset) { 1345 time_t const t = time(NULL); 1346 U32 const h = XXH32(&t, sizeof(t), 1); 1347 seed = h % 10000; 1348 } 1349 DISPLAY("Seed = %u\n", seed); 1350 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); 1351 1352 nbTests += (nbTests==0); /* avoid zero */ 1353 1354 if (testNb==0) result = basicTests(seed, ((double)proba) / 100); 1355 if (result) return 1; 1356 return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration); 1357} 1358