1// LzmaEncoder.cpp
2
3#include "StdAfx.h"
4
5#include "../../../C/Alloc.h"
6
7#include "../Common/CWrappers.h"
8#include "../Common/StreamUtils.h"
9
10#include "LzmaEncoder.h"
11
12// #define LOG_LZMA_THREADS
13
14#ifdef LOG_LZMA_THREADS
15
16#include <stdio.h>
17
18#include "../../Common/IntToString.h"
19#include "../../Windows/TimeUtils.h"
20
21EXTERN_C_BEGIN
22void LzmaEnc_GetLzThreads(CLzmaEncHandle pp, HANDLE lz_threads[2]);
23EXTERN_C_END
24
25#endif
26
27namespace NCompress {
28namespace NLzma {
29
30CEncoder::CEncoder()
31{
32  _encoder = NULL;
33  _encoder = LzmaEnc_Create(&g_AlignedAlloc);
34  if (!_encoder)
35    throw 1;
36}
37
38CEncoder::~CEncoder()
39{
40  if (_encoder)
41    LzmaEnc_Destroy(_encoder, &g_AlignedAlloc, &g_BigAlloc);
42}
43
44static inline wchar_t GetLowCharFast(wchar_t c)
45{
46  return c |= 0x20;
47}
48
49static int ParseMatchFinder(const wchar_t *s, int *btMode, int *numHashBytes)
50{
51  const wchar_t c = GetLowCharFast(*s++);
52  if (c == 'h')
53  {
54    if (GetLowCharFast(*s++) != 'c')
55      return 0;
56    const int num = (int)(*s++ - L'0');
57    if (num < 4 || num > 5)
58      return 0;
59    if (*s != 0)
60      return 0;
61    *btMode = 0;
62    *numHashBytes = num;
63    return 1;
64  }
65
66  if (c != 'b')
67    return 0;
68  {
69    if (GetLowCharFast(*s++) != 't')
70      return 0;
71    const int num = (int)(*s++ - L'0');
72    if (num < 2 || num > 5)
73      return 0;
74    if (*s != 0)
75      return 0;
76    *btMode = 1;
77    *numHashBytes = num;
78    return 1;
79  }
80}
81
82#define SET_PROP_32(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = (int)v; break;
83#define SET_PROP_32U(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = v; break;
84
85HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep);
86HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep)
87{
88  if (propID == NCoderPropID::kMatchFinder)
89  {
90    if (prop.vt != VT_BSTR)
91      return E_INVALIDARG;
92    return ParseMatchFinder(prop.bstrVal, &ep.btMode, &ep.numHashBytes) ? S_OK : E_INVALIDARG;
93  }
94
95  if (propID == NCoderPropID::kAffinity)
96  {
97    if (prop.vt == VT_UI8)
98      ep.affinity = prop.uhVal.QuadPart;
99    else
100      return E_INVALIDARG;
101    return S_OK;
102  }
103
104  if (propID == NCoderPropID::kHashBits)
105  {
106    if (prop.vt == VT_UI4)
107      ep.numHashOutBits = prop.ulVal;
108    else
109      return E_INVALIDARG;
110    return S_OK;
111  }
112
113  if (propID > NCoderPropID::kReduceSize)
114    return S_OK;
115
116  if (propID == NCoderPropID::kReduceSize)
117  {
118    if (prop.vt == VT_UI8)
119      ep.reduceSize = prop.uhVal.QuadPart;
120    else
121      return E_INVALIDARG;
122    return S_OK;
123  }
124
125  if (propID == NCoderPropID::kDictionarySize)
126  {
127    if (prop.vt == VT_UI8)
128    {
129      // 21.03 : we support 64-bit VT_UI8 for dictionary and (dict == 4 GiB)
130      const UInt64 v = prop.uhVal.QuadPart;
131      if (v > ((UInt64)1 << 32))
132        return E_INVALIDARG;
133      UInt32 dict;
134      if (v == ((UInt64)1 << 32))
135        dict = (UInt32)(Int32)-1;
136      else
137        dict = (UInt32)v;
138      ep.dictSize = dict;
139      return S_OK;
140    }
141  }
142
143  if (prop.vt != VT_UI4)
144    return E_INVALIDARG;
145  const UInt32 v = prop.ulVal;
146  switch (propID)
147  {
148    case NCoderPropID::kDefaultProp:
149      if (v > 32)
150        return E_INVALIDARG;
151      ep.dictSize = (v == 32) ? (UInt32)(Int32)-1 : (UInt32)1 << (unsigned)v;
152      break;
153    SET_PROP_32(kLevel, level)
154    SET_PROP_32(kNumFastBytes, fb)
155    SET_PROP_32U(kMatchFinderCycles, mc)
156    SET_PROP_32(kAlgorithm, algo)
157    SET_PROP_32U(kDictionarySize, dictSize)
158    SET_PROP_32(kPosStateBits, pb)
159    SET_PROP_32(kLitPosBits, lp)
160    SET_PROP_32(kLitContextBits, lc)
161    SET_PROP_32(kNumThreads, numThreads)
162    default: return E_INVALIDARG;
163  }
164  return S_OK;
165}
166
167Z7_COM7F_IMF(CEncoder::SetCoderProperties(const PROPID *propIDs,
168    const PROPVARIANT *coderProps, UInt32 numProps))
169{
170  CLzmaEncProps props;
171  LzmaEncProps_Init(&props);
172
173  for (UInt32 i = 0; i < numProps; i++)
174  {
175    const PROPVARIANT &prop = coderProps[i];
176    const PROPID propID = propIDs[i];
177    switch (propID)
178    {
179      case NCoderPropID::kEndMarker:
180        if (prop.vt != VT_BOOL)
181          return E_INVALIDARG;
182        props.writeEndMark = (prop.boolVal != VARIANT_FALSE);
183        break;
184      default:
185        RINOK(SetLzmaProp(propID, prop, props))
186    }
187  }
188  return SResToHRESULT(LzmaEnc_SetProps(_encoder, &props));
189}
190
191
192Z7_COM7F_IMF(CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs,
193    const PROPVARIANT *coderProps, UInt32 numProps))
194{
195  for (UInt32 i = 0; i < numProps; i++)
196  {
197    const PROPVARIANT &prop = coderProps[i];
198    const PROPID propID = propIDs[i];
199    if (propID == NCoderPropID::kExpectedDataSize)
200      if (prop.vt == VT_UI8)
201        LzmaEnc_SetDataSize(_encoder, prop.uhVal.QuadPart);
202  }
203  return S_OK;
204}
205
206
207Z7_COM7F_IMF(CEncoder::WriteCoderProperties(ISequentialOutStream *outStream))
208{
209  Byte props[LZMA_PROPS_SIZE];
210  SizeT size = LZMA_PROPS_SIZE;
211  RINOK(LzmaEnc_WriteProperties(_encoder, props, &size))
212  return WriteStream(outStream, props, size);
213}
214
215
216#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \
217  if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes;
218
219
220
221#ifdef LOG_LZMA_THREADS
222
223static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; }
224
225static void PrintNum(UInt64 val, unsigned numDigits, char c = ' ')
226{
227  char temp[64];
228  char *p = temp + 32;
229  ConvertUInt64ToString(val, p);
230  unsigned len = (unsigned)strlen(p);
231  for (; len < numDigits; len++)
232    *--p = c;
233  printf("%s", p);
234}
235
236static void PrintTime(const char *s, UInt64 val, UInt64 total)
237{
238  printf("  %s :", s);
239  const UInt32 kFreq = 10000000;
240  UInt64 sec = val / kFreq;
241  PrintNum(sec, 6);
242  printf(" .");
243  UInt32 ms = (UInt32)(val - (sec * kFreq)) / (kFreq / 1000);
244  PrintNum(ms, 3, '0');
245
246  while (val > ((UInt64)1 << 56))
247  {
248    val >>= 1;
249    total >>= 1;
250  }
251
252  UInt64 percent = 0;
253  if (total != 0)
254    percent = val * 100 / total;
255  printf("  =");
256  PrintNum(percent, 4);
257  printf("%%");
258}
259
260
261struct CBaseStat
262{
263  UInt64 kernelTime, userTime;
264
265  BOOL Get(HANDLE thread, const CBaseStat *prevStat)
266  {
267    FILETIME creationTimeFT, exitTimeFT, kernelTimeFT, userTimeFT;
268    BOOL res = GetThreadTimes(thread
269      , &creationTimeFT, &exitTimeFT, &kernelTimeFT, &userTimeFT);
270    if (res)
271    {
272      kernelTime = GetTime64(kernelTimeFT);
273      userTime = GetTime64(userTimeFT);
274      if (prevStat)
275      {
276        kernelTime -= prevStat->kernelTime;
277        userTime -= prevStat->userTime;
278      }
279    }
280    return res;
281  }
282};
283
284
285static void PrintStat(HANDLE thread, UInt64 totalTime, const CBaseStat *prevStat)
286{
287  CBaseStat newStat;
288  if (!newStat.Get(thread, prevStat))
289    return;
290
291  PrintTime("K", newStat.kernelTime, totalTime);
292
293  const UInt64 processTime = newStat.kernelTime + newStat.userTime;
294
295  PrintTime("U", newStat.userTime, totalTime);
296  PrintTime("S", processTime, totalTime);
297  printf("\n");
298  // PrintTime("G ", totalTime, totalTime);
299}
300
301#endif
302
303
304
305Z7_COM7F_IMF(CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
306    const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress))
307{
308  CSeqInStreamWrap inWrap;
309  CSeqOutStreamWrap outWrap;
310  CCompressProgressWrap progressWrap;
311
312  inWrap.Init(inStream);
313  outWrap.Init(outStream);
314  progressWrap.Init(progress);
315
316  #ifdef LOG_LZMA_THREADS
317
318  FILETIME startTimeFT;
319  NWindows::NTime::GetCurUtcFileTime(startTimeFT);
320  UInt64 totalTime = GetTime64(startTimeFT);
321  CBaseStat oldStat;
322  if (!oldStat.Get(GetCurrentThread(), NULL))
323    return E_FAIL;
324
325  #endif
326
327
328  SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt,
329      progress ? &progressWrap.vt : NULL, &g_AlignedAlloc, &g_BigAlloc);
330
331  _inputProcessed = inWrap.Processed;
332
333  RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ)
334  RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE)
335  RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS)
336
337
338  #ifdef LOG_LZMA_THREADS
339
340  NWindows::NTime::GetCurUtcFileTime(startTimeFT);
341  totalTime = GetTime64(startTimeFT) - totalTime;
342  HANDLE lz_threads[2];
343  LzmaEnc_GetLzThreads(_encoder, lz_threads);
344  printf("\n");
345  printf("Main: ");  PrintStat(GetCurrentThread(), totalTime, &oldStat);
346  printf("Hash: ");  PrintStat(lz_threads[0], totalTime, NULL);
347  printf("BinT: ");  PrintStat(lz_threads[1], totalTime, NULL);
348  // PrintTime("Total: ", totalTime, totalTime);
349  printf("\n");
350
351  #endif
352
353  return SResToHRESULT(res);
354}
355
356}}
357