1/* LZ4frame API example : compress a file 2 * Modified from an example code by Zbigniew Jędrzejewski-Szmek 3 * 4 * This example streams an input file into an output file 5 * using a bounded memory budget. 6 * Input is read in chunks of IN_CHUNK_SIZE */ 7 8#include <stdio.h> 9#include <stdlib.h> 10#include <string.h> 11#include <errno.h> 12#include <assert.h> 13 14#include <getopt.h> 15#include <lz4frame.h> 16#include <lz4frame_static.h> 17 18#define IN_CHUNK_SIZE (16*1024) 19 20static const LZ4F_preferences_t kPrefs = { 21 { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 22 0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum }, 23 0, /* compression level; 0 == default */ 24 0, /* autoflush */ 25 0, /* favor decompression speed */ 26 { 0, 0, 0 }, /* reserved, must be set to 0 */ 27}; 28 29 30/* safe_fwrite() : 31 * performs fwrite(), ensure operation success, or immediately exit() */ 32static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f) 33{ 34 size_t const writtenSize = fwrite(buf, eltSize, nbElt, f); 35 size_t const expectedSize = eltSize * nbElt; 36 if (nbElt>0) assert(expectedSize / nbElt == eltSize); /* check overflow */ 37 if (writtenSize < expectedSize) { 38 if (ferror(f)) /* note : ferror() must follow fwrite */ 39 fprintf(stderr, "Write failed \n"); 40 else 41 fprintf(stderr, "Write too short \n"); 42 exit(1); 43 } 44} 45 46 47/* ================================================= */ 48/* Streaming Compression example */ 49/* ================================================= */ 50 51typedef struct { 52 int error; 53 unsigned long long size_in; 54 unsigned long long size_out; 55} compressResult_t; 56 57static compressResult_t 58compress_file_internal(FILE* f_in, FILE* f_out, 59 LZ4F_compressionContext_t ctx, 60 void* inBuff, size_t inChunkSize, 61 void* outBuff, size_t outCapacity, 62 FILE* f_unc, long uncOffset) 63{ 64 compressResult_t result = { 1, 0, 0 }; /* result for an error */ 65 long long count_in = 0, count_out, bytesToOffset = -1; 66 67 assert(f_in != NULL); assert(f_out != NULL); 68 assert(ctx != NULL); 69 assert(outCapacity >= LZ4F_HEADER_SIZE_MAX); 70 assert(outCapacity >= LZ4F_compressBound(inChunkSize, &kPrefs)); 71 72 /* write frame header */ 73 { size_t const headerSize = LZ4F_compressBegin(ctx, outBuff, outCapacity, &kPrefs); 74 if (LZ4F_isError(headerSize)) { 75 printf("Failed to start compression: error %u \n", (unsigned)headerSize); 76 return result; 77 } 78 count_out = headerSize; 79 printf("Buffer size is %u bytes, header size %u bytes \n", 80 (unsigned)outCapacity, (unsigned)headerSize); 81 safe_fwrite(outBuff, 1, headerSize, f_out); 82 } 83 84 /* stream file */ 85 for (;;) { 86 size_t compressedSize; 87 long long inSize = IN_CHUNK_SIZE; 88 if (uncOffset >= 0) { 89 bytesToOffset = uncOffset - count_in; 90 91 /* read only remaining bytes to offset position */ 92 if (bytesToOffset < IN_CHUNK_SIZE && bytesToOffset > 0) { 93 inSize = bytesToOffset; 94 } 95 } 96 97 /* input data is at uncompressed data offset */ 98 if (bytesToOffset <= 0 && uncOffset >= 0 && f_unc) { 99 size_t const readSize = fread(inBuff, 1, inSize, f_unc); 100 if (readSize == 0) { 101 uncOffset = -1; 102 continue; 103 } 104 count_in += readSize; 105 compressedSize = LZ4F_uncompressedUpdate(ctx, 106 outBuff, outCapacity, 107 inBuff, readSize, 108 NULL); 109 } else { 110 size_t const readSize = fread(inBuff, 1, inSize, f_in); 111 if (readSize == 0) break; /* nothing left to read from input file */ 112 count_in += readSize; 113 compressedSize = LZ4F_compressUpdate(ctx, 114 outBuff, outCapacity, 115 inBuff, readSize, 116 NULL); 117 118 } 119 120 if (LZ4F_isError(compressedSize)) { 121 printf("Compression failed: error %u \n", (unsigned)compressedSize); 122 return result; 123 } 124 125 printf("Writing %u bytes\n", (unsigned)compressedSize); 126 safe_fwrite(outBuff, 1, compressedSize, f_out); 127 count_out += compressedSize; 128 } 129 130 /* flush whatever remains within internal buffers */ 131 { size_t const compressedSize = LZ4F_compressEnd(ctx, 132 outBuff, outCapacity, 133 NULL); 134 if (LZ4F_isError(compressedSize)) { 135 printf("Failed to end compression: error %u \n", (unsigned)compressedSize); 136 return result; 137 } 138 139 printf("Writing %u bytes \n", (unsigned)compressedSize); 140 safe_fwrite(outBuff, 1, compressedSize, f_out); 141 count_out += compressedSize; 142 } 143 144 result.size_in = count_in; 145 result.size_out = count_out; 146 result.error = 0; 147 return result; 148} 149 150static compressResult_t 151compress_file(FILE* f_in, FILE* f_out, 152 FILE* f_unc, int uncOffset) 153{ 154 assert(f_in != NULL); 155 assert(f_out != NULL); 156 157 /* resource allocation */ 158 LZ4F_compressionContext_t ctx; 159 size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); 160 void* const src = malloc(IN_CHUNK_SIZE); 161 size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs); /* large enough for any input <= IN_CHUNK_SIZE */ 162 void* const outbuff = malloc(outbufCapacity); 163 164 compressResult_t result = { 1, 0, 0 }; /* == error (default) */ 165 if (!LZ4F_isError(ctxCreation) && src && outbuff) { 166 result = compress_file_internal(f_in, f_out, 167 ctx, 168 src, IN_CHUNK_SIZE, 169 outbuff, outbufCapacity, 170 f_unc, uncOffset); 171 } else { 172 printf("error : resource allocation failed \n"); 173 } 174 175 LZ4F_freeCompressionContext(ctx); /* supports free on NULL */ 176 free(src); 177 free(outbuff); 178 return result; 179} 180 181 182/* ================================================= */ 183/* Streaming decompression example */ 184/* ================================================= */ 185 186static size_t get_block_size(const LZ4F_frameInfo_t* info) { 187 switch (info->blockSizeID) { 188 case LZ4F_default: 189 case LZ4F_max64KB: return 1 << 16; 190 case LZ4F_max256KB: return 1 << 18; 191 case LZ4F_max1MB: return 1 << 20; 192 case LZ4F_max4MB: return 1 << 22; 193 default: 194 printf("Impossible with expected frame specification (<=v1.6.1)\n"); 195 exit(1); 196 } 197} 198 199/* @return : 1==error, 0==success */ 200static int 201decompress_file_internal(FILE* f_in, FILE* f_out, 202 LZ4F_dctx* dctx, 203 void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed, 204 void* dst, size_t dstCapacity) 205{ 206 int firstChunk = 1; 207 size_t ret = 1; 208 209 assert(f_in != NULL); assert(f_out != NULL); 210 assert(dctx != NULL); 211 assert(src != NULL); assert(srcCapacity > 0); assert(filled <= srcCapacity); assert(alreadyConsumed <= filled); 212 assert(dst != NULL); assert(dstCapacity > 0); 213 214 /* Decompression */ 215 while (ret != 0) { 216 /* Load more input */ 217 size_t readSize = firstChunk ? filled : fread(src, 1, srcCapacity, f_in); firstChunk=0; 218 const void* srcPtr = (const char*)src + alreadyConsumed; alreadyConsumed=0; 219 const void* const srcEnd = (const char*)srcPtr + readSize; 220 if (readSize == 0 || ferror(f_in)) { 221 printf("Decompress: not enough input or error reading file\n"); 222 return 1; 223 } 224 225 /* Decompress: 226 * Continue while there is more input to read (srcPtr != srcEnd) 227 * and the frame isn't over (ret != 0) 228 */ 229 while (srcPtr < srcEnd && ret != 0) { 230 /* Any data within dst has been flushed at this stage */ 231 size_t dstSize = dstCapacity; 232 size_t srcSize = (const char*)srcEnd - (const char*)srcPtr; 233 ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL); 234 if (LZ4F_isError(ret)) { 235 printf("Decompression error: %s\n", LZ4F_getErrorName(ret)); 236 return 1; 237 } 238 /* Flush output */ 239 if (dstSize != 0) safe_fwrite(dst, 1, dstSize, f_out); 240 /* Update input */ 241 srcPtr = (const char*)srcPtr + srcSize; 242 } 243 244 assert(srcPtr <= srcEnd); 245 246 /* Ensure all input data has been consumed. 247 * It is valid to have multiple frames in the same file, 248 * but this example only supports one frame. 249 */ 250 if (srcPtr < srcEnd) { 251 printf("Decompress: Trailing data left in file after frame\n"); 252 return 1; 253 } 254 } 255 256 /* Check that there isn't trailing data in the file after the frame. 257 * It is valid to have multiple frames in the same file, 258 * but this example only supports one frame. 259 */ 260 { size_t const readSize = fread(src, 1, 1, f_in); 261 if (readSize != 0 || !feof(f_in)) { 262 printf("Decompress: Trailing data left in file after frame\n"); 263 return 1; 264 } } 265 266 return 0; 267} 268 269 270/* @return : 1==error, 0==completed */ 271static int 272decompress_file_allocDst(FILE* f_in, FILE* f_out, 273 LZ4F_dctx* dctx, 274 void* src, size_t srcCapacity) 275{ 276 assert(f_in != NULL); assert(f_out != NULL); 277 assert(dctx != NULL); 278 assert(src != NULL); 279 assert(srcCapacity >= LZ4F_HEADER_SIZE_MAX); /* ensure LZ4F_getFrameInfo() can read enough data */ 280 281 /* Read Frame header */ 282 size_t const readSize = fread(src, 1, srcCapacity, f_in); 283 if (readSize == 0 || ferror(f_in)) { 284 printf("Decompress: not enough input or error reading file\n"); 285 return 1; 286 } 287 288 LZ4F_frameInfo_t info; 289 size_t consumedSize = readSize; 290 { size_t const fires = LZ4F_getFrameInfo(dctx, &info, src, &consumedSize); 291 if (LZ4F_isError(fires)) { 292 printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(fires)); 293 return 1; 294 } } 295 296 /* Allocating enough space for an entire block isn't necessary for 297 * correctness, but it allows some memcpy's to be elided. 298 */ 299 size_t const dstCapacity = get_block_size(&info); 300 void* const dst = malloc(dstCapacity); 301 if (!dst) { perror("decompress_file(dst)"); return 1; } 302 303 int const decompressionResult = decompress_file_internal( 304 f_in, f_out, 305 dctx, 306 src, srcCapacity, readSize-consumedSize, consumedSize, 307 dst, dstCapacity); 308 309 free(dst); 310 return decompressionResult; 311} 312 313 314/* @result : 1==error, 0==success */ 315static int decompress_file(FILE* f_in, FILE* f_out) 316{ 317 assert(f_in != NULL); assert(f_out != NULL); 318 319 /* Resource allocation */ 320 void* const src = malloc(IN_CHUNK_SIZE); 321 if (!src) { perror("decompress_file(src)"); return 1; } 322 323 LZ4F_dctx* dctx; 324 { size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); 325 if (LZ4F_isError(dctxStatus)) { 326 printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus)); 327 } } 328 329 int const result = !dctx ? 1 /* error */ : 330 decompress_file_allocDst(f_in, f_out, dctx, src, IN_CHUNK_SIZE); 331 332 free(src); 333 LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */ 334 return result; 335} 336 337 338int compareFiles(FILE* fp0, FILE* fp1, FILE* fpUnc, long uncOffset) 339{ 340 int result = 0; 341 long bytesRead = 0; 342 long bytesToOffset = -1; 343 long b1Size = 1024; 344 345 while (result==0) { 346 char b1[b1Size]; 347 size_t r1; 348 size_t bytesToRead = sizeof b1; 349 if (uncOffset >= 0) { 350 bytesToOffset = uncOffset - bytesRead; 351 352 /* read remainder to offset */ 353 if (bytesToOffset < b1Size) { 354 bytesToRead = bytesToOffset; 355 } 356 } 357 358 char b0[1024]; 359 size_t r0; 360 if (bytesToOffset <= 0 && fpUnc) { 361 bytesToRead = sizeof b1; 362 r0 = fread(b0, 1,bytesToRead, fpUnc); 363 } else { 364 r0 = fread(b0, 1, bytesToRead, fp0); 365 } 366 367 r1 = fread(b1, 1, r0, fp1); 368 369 result = (r0 != r1); 370 if (!r0 || !r1) break; 371 if (!result) result = memcmp(b0, b1, r0); 372 373 bytesRead += r1; 374 } 375 376 return result; 377} 378 379 380int main(int argc, char **argv) { 381 char inpFilename[256] = { 0 }; 382 char lz4Filename[256] = { 0 }; 383 char decFilename[256] = { 0 }; 384 385 int uncOffset = -1; 386 char uncFilename[256] = { 0 }; 387 int opt; 388 389 if (argc < 2) { 390 printf("Please specify input filename\n"); 391 return EXIT_FAILURE; 392 } 393 394 snprintf(inpFilename, 256, "%s", argv[1]); 395 snprintf(lz4Filename, 256, "%s.lz4", argv[1]); 396 snprintf(decFilename, 256, "%s.lz4.dec", argv[1]); 397 398 while ((opt = getopt(argc, argv, "o:d:")) != -1) { 399 switch (opt) { 400 case 'd': 401 snprintf(uncFilename, 256, "%s", optarg); 402 break; 403 case 'o': 404 uncOffset = atoi(optarg); 405 break; 406 default: 407 printf("usage: %s <input file> [-o <offset> -d <file>]\n", argv[0]); 408 printf("-o uncompressed data offset\n"); 409 printf(" inject uncompressed data at this offset into the lz4 file\n"); 410 printf("-d uncompressed file\n"); 411 printf(" file to inject without compression into the lz4 file\n"); 412 return EXIT_FAILURE; 413 } 414 } 415 416 printf("inp = [%s]\n", inpFilename); 417 printf("lz4 = [%s]\n", lz4Filename); 418 printf("dec = [%s]\n", decFilename); 419 if (uncOffset > 0) { 420 printf("unc = [%s]\n", uncFilename); 421 printf("ofs = [%i]\n", uncOffset); 422 } 423 424 /* compress */ 425 { FILE* const inpFp = fopen(inpFilename, "rb"); 426 FILE* const outFp = fopen(lz4Filename, "wb"); 427 FILE* const uncFp = fopen(uncFilename, "rb"); 428 429 printf("compress : %s -> %s\n", inpFilename, lz4Filename); 430 compressResult_t const ret = compress_file( 431 inpFp, outFp, 432 uncFp, uncOffset); 433 434 fclose(outFp); 435 fclose(inpFp); 436 if (uncFp) 437 fclose(uncFp); 438 439 if (ret.error) { 440 printf("compress : failed with code %i\n", ret.error); 441 return ret.error; 442 } 443 printf("%s: %zu → %zu bytes, %.1f%%\n", 444 inpFilename, 445 (size_t)ret.size_in, (size_t)ret.size_out, /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */ 446 (double)ret.size_out / ret.size_in * 100); 447 printf("compress : done\n"); 448 } 449 450 /* decompress */ 451 { FILE* const inpFp = fopen(lz4Filename, "rb"); 452 FILE* const outFp = fopen(decFilename, "wb"); 453 454 printf("decompress : %s -> %s\n", lz4Filename, decFilename); 455 int const ret = decompress_file(inpFp, outFp); 456 457 fclose(outFp); 458 fclose(inpFp); 459 460 if (ret) { 461 printf("decompress : failed with code %i\n", ret); 462 return ret; 463 } 464 printf("decompress : done\n"); 465 } 466 467 /* verify */ 468 { FILE* const inpFp = fopen(inpFilename, "rb"); 469 FILE* const decFp = fopen(decFilename, "rb"); 470 FILE* const uncFp = fopen(uncFilename, "rb"); 471 472 printf("verify : %s <-> %s\n", inpFilename, decFilename); 473 int const cmp = compareFiles(inpFp, decFp, 474 uncFp, uncOffset); 475 476 fclose(decFp); 477 fclose(inpFp); 478 if (uncFp) 479 fclose(uncFp); 480 481 if (cmp) { 482 printf("corruption detected : decompressed file differs from original\n"); 483 return cmp; 484 } 485 printf("verify : OK\n"); 486 } 487 488 return 0; 489} 490