1 // ZstdHandler.cpp
2
3 #include "StdAfx.h"
4
5 // #define Z7_USE_ZSTD_ORIG_DECODER
6 // #define Z7_USE_ZSTD_COMPRESSION
7
8 #include "../../Common/ComTry.h"
9
10 #include "../Common/MethodProps.h"
11 #include "../Common/ProgressUtils.h"
12 #include "../Common/RegisterArc.h"
13 #include "../Common/StreamUtils.h"
14
15 #include "../Compress/CopyCoder.h"
16 #include "../Compress/ZstdDecoder.h"
17 #ifdef Z7_USE_ZSTD_ORIG_DECODER
18 #include "../Compress/Zstd2Decoder.h"
19 #endif
20
21 #ifdef Z7_USE_ZSTD_COMPRESSION
22 #include "../Compress/ZstdEncoder.h"
23 #include "../Compress/ZstdEncoderProps.h"
24 #include "Common/HandlerOut.h"
25 #endif
26
27 #include "Common/DummyOutStream.h"
28
29 #include "../../../C/CpuArch.h"
30
31 using namespace NWindows;
32
33 namespace NArchive {
34 namespace NZstd {
35
36 #define DESCRIPTOR_Get_DictionaryId_Flag(d) ((d) & 3)
37 #define DESCRIPTOR_FLAG_CHECKSUM (1 << 2)
38 #define DESCRIPTOR_FLAG_RESERVED (1 << 3)
39 #define DESCRIPTOR_FLAG_UNUSED (1 << 4)
40 #define DESCRIPTOR_FLAG_SINGLE (1 << 5)
41 #define DESCRIPTOR_Get_ContentSize_Flag3(d) ((d) >> 5)
42 #define DESCRIPTOR_Is_ContentSize_Defined(d) (((d) & 0xe0) != 0)
43
44 struct CFrameHeader
45 {
46 Byte Descriptor;
47 Byte WindowDescriptor;
48 UInt32 DictionaryId;
49 UInt64 ContentSize;
50
51 /* by zstd specification:
52 the decoder must check that (Is_Reserved() == false)
53 the decoder must ignore Unused_bit */
Is_ReservedNArchive::NZstd::CFrameHeader54 bool Is_Reserved() const { return (Descriptor & DESCRIPTOR_FLAG_RESERVED) != 0; }
Is_ChecksumNArchive::NZstd::CFrameHeader55 bool Is_Checksum() const { return (Descriptor & DESCRIPTOR_FLAG_CHECKSUM) != 0; }
Is_SingleSegmentNArchive::NZstd::CFrameHeader56 bool Is_SingleSegment() const { return (Descriptor & DESCRIPTOR_FLAG_SINGLE) != 0; }
Is_ContentSize_DefinedNArchive::NZstd::CFrameHeader57 bool Is_ContentSize_Defined() const { return DESCRIPTOR_Is_ContentSize_Defined(Descriptor); }
Get_DictionaryId_FlagNArchive::NZstd::CFrameHeader58 unsigned Get_DictionaryId_Flag() const { return DESCRIPTOR_Get_DictionaryId_Flag(Descriptor); }
Get_ContentSize_Flag3NArchive::NZstd::CFrameHeader59 unsigned Get_ContentSize_Flag3() const { return DESCRIPTOR_Get_ContentSize_Flag3(Descriptor); }
60
ParseNArchive::NZstd::CFrameHeader61 const Byte *Parse(const Byte *p, int size)
62 {
63 if ((unsigned)size < 2)
64 return NULL;
65 Descriptor = *p++;
66 size--;
67 {
68 Byte w = 0;
69 if (!Is_SingleSegment())
70 {
71 w = *p++;
72 size--;
73 }
74 WindowDescriptor = w;
75 }
76 {
77 unsigned n = Get_DictionaryId_Flag();
78 UInt32 d = 0;
79 if (n)
80 {
81 n = (unsigned)1 << (n - 1);
82 if ((size -= (int)n) < 0)
83 return NULL;
84 d = GetUi32(p) & ((UInt32)(Int32)-1 >> (32 - 8u * n));
85 p += n;
86 }
87 DictionaryId = d;
88 }
89 {
90 unsigned n = Get_ContentSize_Flag3();
91 UInt64 v = 0;
92 if (n)
93 {
94 n >>= 1;
95 if (n == 1)
96 v = 256;
97 n = (unsigned)1 << n;
98 if ((size -= (int)n) < 0)
99 return NULL;
100 v += GetUi64(p) & ((UInt64)(Int64)-1 >> (64 - 8u * n));
101 p += n;
102 }
103 ContentSize = v;
104 }
105 return p;
106 }
107 };
108
109
110
111 class CHandler Z7_final:
112 public IInArchive,
113 public IArchiveOpenSeq,
114 public ISetProperties,
115 #ifdef Z7_USE_ZSTD_COMPRESSION
116 public IOutArchive,
117 #endif
118 public CMyUnknownImp
119 {
120 Z7_COM_QI_BEGIN2(IInArchive)
121 Z7_COM_QI_ENTRY(IArchiveOpenSeq)
122 Z7_COM_QI_ENTRY(ISetProperties)
123 #ifdef Z7_USE_ZSTD_COMPRESSION
124 Z7_COM_QI_ENTRY(IOutArchive)
125 #endif
126 Z7_COM_QI_END
127 Z7_COM_ADDREF_RELEASE
128
129 Z7_IFACE_COM7_IMP(IInArchive)
130 Z7_IFACE_COM7_IMP(IArchiveOpenSeq)
131 Z7_IFACE_COM7_IMP(ISetProperties)
132 #ifdef Z7_USE_ZSTD_COMPRESSION
133 Z7_IFACE_COM7_IMP(IOutArchive)
134 #endif
135
136 bool _isArc;
137 bool _needSeekToStart;
138 // bool _dataAfterEnd;
139 // bool _needMoreInput;
140 bool _unsupportedBlock;
141
142 bool _wasParsed;
143 bool _phySize_Decoded_Defined;
144 bool _unpackSize_Defined; // decoded
145 bool _decoded_Info_Defined;
146
147 bool _parseMode;
148 bool _disableHash;
149 // bool _smallMode;
150
151 UInt64 _phySize;
152 UInt64 _phySize_Decoded;
153 UInt64 _unpackSize;
154
155 CZstdDecInfo _parsed_Info;
156 CZstdDecInfo _decoded_Info;
157
158 CMyComPtr<IInStream> _stream;
159 CMyComPtr<ISequentialInStream> _seqStream;
160
161 #ifdef Z7_USE_ZSTD_COMPRESSION
162 CSingleMethodProps _props;
163 #endif
164
165 public:
CHandler()166 CHandler():
167 _parseMode(false),
168 _disableHash(false)
169 // _smallMode(false)
170 {}
171 };
172
173
174 static const Byte kProps[] =
175 {
176 kpidSize,
177 kpidPackSize
178 };
179
180 static const Byte kArcProps[] =
181 {
182 kpidNumStreams,
183 kpidNumBlocks,
184 kpidMethod,
185 // kpidChecksum
186 kpidCRC
187 };
188
189 IMP_IInArchive_Props
190 IMP_IInArchive_ArcProps
191
192
193 // static const unsigned kBlockType_Raw = 0;
194 static const unsigned kBlockType_RLE = 1;
195 // static const unsigned kBlockType_Compressed = 2;
196 static const unsigned kBlockType_Reserved = 3;
197 /*
198 static const char * const kNames[] =
199 {
200 "RAW"
201 , "RLE"
202 , "Compressed"
203 , "Reserved"
204 };
205 */
206
Add_UInt64(AString & s,const char * name,UInt64 v)207 static void Add_UInt64(AString &s, const char *name, UInt64 v)
208 {
209 s.Add_OptSpaced(name);
210 s.Add_Colon();
211 s.Add_UInt64(v);
212 }
213
214
PrintSize(AString & s,UInt64 w)215 static void PrintSize(AString &s, UInt64 w)
216 {
217 char c = 0;
218 if ((w & ((1 << 30) - 1)) == 0) { c = 'G'; w >>= 30; }
219 else if ((w & ((1 << 20) - 1)) == 0) { c = 'M'; w >>= 20; }
220 else if ((w & ((1 << 10) - 1)) == 0) { c = 'K'; w >>= 10; }
221 s.Add_UInt64(w);
222 if (c)
223 {
224 s.Add_Char(c);
225 s += "iB";
226 }
227 }
228
229
Z7_COM7F_IMF(CHandler::GetArchiveProperty (PROPID propID,PROPVARIANT * value))230 Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value))
231 {
232 NCOM::CPropVariant prop;
233
234 CZstdDecInfo *p = NULL;
235 if (_wasParsed || !_decoded_Info_Defined)
236 p = &_parsed_Info;
237 else if (_decoded_Info_Defined)
238 p = &_decoded_Info;
239
240 switch (propID)
241 {
242 case kpidPhySize:
243 if (_wasParsed)
244 prop = _phySize;
245 else if (_phySize_Decoded_Defined)
246 prop = _phySize_Decoded;
247 break;
248
249 case kpidUnpackSize:
250 if (_unpackSize_Defined)
251 prop = _unpackSize;
252 break;
253
254 case kpidNumStreams:
255 if (p)
256 if (_wasParsed || _decoded_Info_Defined)
257 prop = p->num_DataFrames;
258 break;
259
260 case kpidNumBlocks:
261 if (p)
262 if (_wasParsed || _decoded_Info_Defined)
263 prop = p->num_Blocks;
264 break;
265
266 // case kpidChecksum:
267 case kpidCRC:
268 if (p)
269 if (p->checksum_Defined && p->num_DataFrames == 1)
270 prop = p->checksum; // it's checksum from last frame
271 break;
272
273 case kpidMethod:
274 {
275 AString s;
276 s.Add_OptSpaced(p == &_decoded_Info ?
277 "decoded:" : _wasParsed ?
278 "parsed:" :
279 "header-open-only:");
280
281 if (p->dictionaryId != 0)
282 {
283 if (p->are_DictionaryId_Different)
284 s.Add_OptSpaced("different-dictionary-IDs");
285 s.Add_OptSpaced("dictionary-ID:");
286 s.Add_UInt32(p->dictionaryId);
287 }
288 /*
289 if (ContentSize_Defined)
290 {
291 s.Add_OptSpaced("ContentSize=");
292 s.Add_UInt64(ContentSize_Total);
293 }
294 */
295 // if (p->are_Checksums)
296 if (p->descriptor_OR & DESCRIPTOR_FLAG_CHECKSUM)
297 s.Add_OptSpaced("XXH64");
298 if (p->descriptor_NOT_OR & DESCRIPTOR_FLAG_CHECKSUM)
299 s.Add_OptSpaced("NO-XXH64");
300
301 if (p->descriptor_OR & DESCRIPTOR_FLAG_UNUSED)
302 s.Add_OptSpaced("unused_bit");
303
304 if (p->descriptor_OR & DESCRIPTOR_FLAG_SINGLE)
305 s.Add_OptSpaced("single-segments");
306
307 if (p->descriptor_NOT_OR & DESCRIPTOR_FLAG_SINGLE)
308 {
309 // Add_UInt64(s, "wnd-descriptors", p->num_WindowDescriptors);
310 s.Add_OptSpaced("wnd-desc-log-MAX:");
311 // WindowDescriptor_MAX = 16 << 3; // for debug
312 const unsigned e = p->windowDescriptor_MAX >> 3;
313 s.Add_UInt32(e + 10);
314 const unsigned m = p->windowDescriptor_MAX & 7;
315 if (m != 0)
316 {
317 s.Add_Dot();
318 s.Add_UInt32(m);
319 }
320 }
321
322 if (DESCRIPTOR_Is_ContentSize_Defined(p->descriptor_OR) ||
323 (p->descriptor_NOT_OR & DESCRIPTOR_FLAG_SINGLE))
324 /*
325 if (p->are_ContentSize_Known ||
326 p->are_WindowDescriptors)
327 */
328 {
329 s.Add_OptSpaced("wnd-MAX:");
330 PrintSize(s, p->windowSize_MAX);
331 if (p->windowSize_MAX != p->windowSize_Allocate_MAX)
332 {
333 s.Add_OptSpaced("wnd-use-MAX:");
334 PrintSize(s, p->windowSize_Allocate_MAX);
335 }
336 }
337
338 if (p->num_DataFrames != 1)
339 Add_UInt64(s, "data-frames", p->num_DataFrames);
340 if (p->num_SkipFrames != 0)
341 {
342 Add_UInt64(s, "skip-frames", p->num_SkipFrames);
343 Add_UInt64(s, "skip-frames-size-total", p->skipFrames_Size);
344 }
345
346 if (p->are_ContentSize_Unknown)
347 s.Add_OptSpaced("unknown-content-size");
348
349 if (DESCRIPTOR_Is_ContentSize_Defined(p->descriptor_OR))
350 {
351 Add_UInt64(s, "content-size-frame-max", p->contentSize_MAX);
352 Add_UInt64(s, "content-size-total", p->contentSize_Total);
353 }
354
355 /*
356 for (unsigned i = 0; i < 4; i++)
357 {
358 const UInt64 n = p->num_Blocks_forType[i];
359 if (n)
360 {
361 s.Add_OptSpaced(kNames[i]);
362 s += "-blocks:";
363 s.Add_UInt64(n);
364
365 s.Add_OptSpaced(kNames[i]);
366 s += "-block-bytes:";
367 s.Add_UInt64(p->num_BlockBytes_forType[i]);
368 }
369 }
370 */
371 prop = s;
372 break;
373 }
374
375 case kpidErrorFlags:
376 {
377 UInt32 v = 0;
378 if (!_isArc) v |= kpv_ErrorFlags_IsNotArc;
379 // if (_needMoreInput) v |= kpv_ErrorFlags_UnexpectedEnd;
380 // if (_dataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd;
381 if (_unsupportedBlock) v |= kpv_ErrorFlags_UnsupportedMethod;
382 /*
383 if (_parsed_Info.numBlocks_forType[kBlockType_Reserved])
384 v |= kpv_ErrorFlags_UnsupportedMethod;
385 */
386 prop = v;
387 break;
388 }
389
390 default: break;
391 }
392 prop.Detach(value);
393 return S_OK;
394 }
395
396
Z7_COM7F_IMF(CHandler::GetNumberOfItems (UInt32 * numItems))397 Z7_COM7F_IMF(CHandler::GetNumberOfItems(UInt32 *numItems))
398 {
399 *numItems = 1;
400 return S_OK;
401 }
402
Z7_COM7F_IMF(CHandler::GetProperty (UInt32,PROPID propID,PROPVARIANT * value))403 Z7_COM7F_IMF(CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value))
404 {
405 NCOM::CPropVariant prop;
406 switch (propID)
407 {
408 case kpidPackSize:
409 if (_wasParsed)
410 prop = _phySize;
411 else if (_phySize_Decoded_Defined)
412 prop = _phySize_Decoded;
413 break;
414
415 case kpidSize:
416 if (_wasParsed && !_parsed_Info.are_ContentSize_Unknown)
417 prop = _parsed_Info.contentSize_Total;
418 else if (_unpackSize_Defined)
419 prop = _unpackSize;
420 break;
421
422 default: break;
423 }
424 prop.Detach(value);
425 return S_OK;
426 }
427
428 static const unsigned kSignatureSize = 4;
429 static const Byte k_Signature[kSignatureSize] = { 0x28, 0xb5, 0x2f, 0xfd } ;
430
431 static const UInt32 kDataFrameSignature32 = 0xfd2fb528;
432 static const UInt32 kSkipFrameSignature = 0x184d2a50;
433 static const UInt32 kSkipFrameSignature_Mask = 0xfffffff0;
434
435 /*
436 API_FUNC_static_IsArc IsArc_Zstd(const Byte *p, size_t size)
437 {
438 if (size < kSignatureSize)
439 return k_IsArc_Res_NEED_MORE;
440 if (memcmp(p, k_Signature, kSignatureSize) != 0)
441 {
442 const UInt32 v = GetUi32(p);
443 if ((v & kSkipFrameSignature_Mask) != kSkipFrameSignature)
444 return k_IsArc_Res_NO;
445 return k_IsArc_Res_YES;
446 }
447 p += 4;
448 // return k_IsArc_Res_YES;
449 }
450 }
451 */
452
453 // kBufSize must be >= (ZSTD_FRAMEHEADERSIZE_MAX = 18)
454 // we use big buffer for fast parsing of worst case small blocks.
455 static const unsigned kBufSize =
456 1 << 9;
457 // 1 << 14; // fastest in real file
458
459 struct CStreamBuffer
460 {
461 unsigned pos;
462 unsigned lim;
463 IInStream *Stream;
464 UInt64 StreamOffset;
465 Byte buf[kBufSize];
466
CStreamBufferNArchive::NZstd::CStreamBuffer467 CStreamBuffer():
468 pos(0),
469 lim(0),
470 StreamOffset(0)
471 {}
AvailNArchive::NZstd::CStreamBuffer472 unsigned Avail() const { return lim - pos; }
GetPtrNArchive::NZstd::CStreamBuffer473 const Byte *GetPtr() const { return &buf[pos]; }
GetCurOffsetNArchive::NZstd::CStreamBuffer474 UInt64 GetCurOffset() const { return StreamOffset - Avail(); }
SkipInBufNArchive::NZstd::CStreamBuffer475 void SkipInBuf(UInt32 size) { pos += size; }
476 HRESULT Skip(UInt32 size);
477 HRESULT Read(unsigned num);
478 };
479
Skip(UInt32 size)480 HRESULT CStreamBuffer::Skip(UInt32 size)
481 {
482 unsigned rem = lim - pos;
483 if (rem != 0)
484 {
485 if (rem > size)
486 rem = size;
487 pos += rem;
488 size -= rem;
489 if (pos != lim)
490 return S_OK;
491 }
492 if (size == 0)
493 return S_OK;
494 return Stream->Seek(size, STREAM_SEEK_CUR, &StreamOffset);
495 }
496
Read(unsigned num)497 HRESULT CStreamBuffer::Read(unsigned num)
498 {
499 if (lim - pos >= num)
500 return S_OK;
501 if (pos != 0)
502 {
503 lim -= pos;
504 memmove(buf, buf + pos, lim);
505 pos = 0;
506 }
507 size_t processed = kBufSize - ((unsigned)StreamOffset & (kBufSize - 1));
508 const unsigned avail = kBufSize - lim;
509 num -= lim;
510 if (avail < processed || processed < num)
511 processed = avail;
512 const HRESULT res = ReadStream(Stream, buf + lim, &processed);
513 StreamOffset += processed;
514 lim += (unsigned)processed;
515 return res;
516 }
517
518
519 static const unsigned k_ZSTD_FRAMEHEADERSIZE_MAX = 4 + 14;
520
Z7_COM7F_IMF(CHandler::Open (IInStream * stream,const UInt64 *,IArchiveOpenCallback * callback))521 Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *callback))
522 {
523 COM_TRY_BEGIN
524 Close();
525
526 CZstdDecInfo *p = &_parsed_Info;
527 // p->are_ContentSize_Unknown = False;
528 CStreamBuffer sb;
529 sb.Stream = stream;
530
531 for (;;)
532 {
533 RINOK(sb.Read(k_ZSTD_FRAMEHEADERSIZE_MAX))
534 if (sb.Avail() < kSignatureSize)
535 break;
536
537 if (callback && (ZstdDecInfo_GET_NUM_FRAMES(p) & 0xFFF) == 2)
538 {
539 const UInt64 numBytes = sb.GetCurOffset();
540 RINOK(callback->SetCompleted(NULL, &numBytes))
541 }
542
543 const UInt32 v = GetUi32(sb.GetPtr());
544 if (v != kDataFrameSignature32)
545 {
546 if ((v & kSkipFrameSignature_Mask) != kSkipFrameSignature)
547 break;
548 _phySize = sb.GetCurOffset() + 8;
549 p->num_SkipFrames++;
550 sb.SkipInBuf(4);
551 if (sb.Avail() < 4)
552 break;
553 const UInt32 size = GetUi32(sb.GetPtr());
554 p->skipFrames_Size += size;
555 sb.SkipInBuf(4);
556 _phySize = sb.GetCurOffset() + size;
557 RINOK(sb.Skip(size))
558 continue;
559 }
560
561 p->num_DataFrames++;
562 // _numStreams_Defined = true;
563 sb.SkipInBuf(4);
564 CFrameHeader fh;
565 {
566 const Byte *data = fh.Parse(sb.GetPtr(), (int)sb.Avail());
567 if (!data)
568 {
569 // _needMoreInput = true;
570 // we set size for one byte more to show that stream was truncated
571 _phySize = sb.StreamOffset + 1;
572 break;
573 }
574 if (fh.Is_Reserved())
575 {
576 // we don't want false detection
577 if (ZstdDecInfo_GET_NUM_FRAMES(p) == 1)
578 return S_FALSE;
579 // _phySize = sb.GetCurOffset();
580 break;
581 }
582 sb.SkipInBuf((unsigned)(data - sb.GetPtr()));
583 }
584
585 p->descriptor_OR = (Byte)(p->descriptor_OR | fh.Descriptor);
586 p->descriptor_NOT_OR = (Byte)(p->descriptor_NOT_OR | ~fh.Descriptor);
587
588 // _numBlocks_Defined = true;
589 // if (fh.Get_DictionaryId_Flag())
590 // p->dictionaryId_Cur = fh.DictionaryId;
591 if (fh.DictionaryId != 0)
592 {
593 if (p->dictionaryId == 0)
594 p->dictionaryId = fh.DictionaryId;
595 else if (p->dictionaryId != fh.DictionaryId)
596 p->are_DictionaryId_Different = True;
597 }
598
599 UInt32 blockSizeAllowedMax = (UInt32)1 << 17;
600 {
601 UInt64 winSize = fh.ContentSize;
602 UInt64 winSize_forAllocate = fh.ContentSize;
603 if (!fh.Is_SingleSegment())
604 {
605 if (p->windowDescriptor_MAX < fh.WindowDescriptor)
606 p->windowDescriptor_MAX = fh.WindowDescriptor;
607 const unsigned e = (fh.WindowDescriptor >> 3);
608 const unsigned m = (fh.WindowDescriptor & 7);
609 winSize = (UInt64)(8 + m) << (e + 10 - 3);
610 if (!fh.Is_ContentSize_Defined()
611 || fh.DictionaryId != 0
612 || winSize_forAllocate > winSize)
613 winSize_forAllocate = winSize;
614 // p->are_WindowDescriptors = true;
615 }
616 else
617 {
618 // p->are_SingleSegments = True;
619 }
620 if (blockSizeAllowedMax > winSize)
621 blockSizeAllowedMax = (UInt32)winSize;
622 if (p->windowSize_MAX < winSize)
623 p->windowSize_MAX = winSize;
624 if (p->windowSize_Allocate_MAX < winSize_forAllocate)
625 p->windowSize_Allocate_MAX = winSize_forAllocate;
626 }
627
628 if (fh.Is_ContentSize_Defined())
629 {
630 // p->are_ContentSize_Known = True;
631 p->contentSize_Total += fh.ContentSize;
632 if (p->contentSize_MAX < fh.ContentSize)
633 p->contentSize_MAX = fh.ContentSize;
634 }
635 else
636 {
637 p->are_ContentSize_Unknown = True;
638 }
639
640 p->checksum_Defined = false;
641
642 // p->numBlocks_forType[3] += 99; // for debug
643
644 if (!_parseMode)
645 {
646 if (ZstdDecInfo_GET_NUM_FRAMES(p) == 1)
647 break;
648 }
649
650 _wasParsed = true;
651
652 bool blocksWereParsed = false;
653
654 for (;;)
655 {
656 if (callback && (p->num_Blocks & 0xFFF) == 2)
657 {
658 // Sleep(10);
659 const UInt64 numBytes = sb.GetCurOffset();
660 RINOK(callback->SetCompleted(NULL, &numBytes))
661 }
662 _phySize = sb.GetCurOffset() + 3;
663 RINOK(sb.Read(3))
664 if (sb.Avail() < 3)
665 {
666 // _needMoreInput = true;
667 // return S_FALSE;
668 break; // change it
669 }
670 const unsigned pos = sb.pos;
671 sb.pos = pos + 3;
672 UInt32 b = 0;
673 b += (UInt32)sb.buf[pos];
674 b += (UInt32)sb.buf[pos + 1] << (8 * 1);
675 b += (UInt32)sb.buf[pos + 2] << (8 * 2);
676 p->num_Blocks++;
677 const unsigned blockType = (b >> 1) & 3;
678 UInt32 size = b >> 3;
679 // p->num_Blocks_forType[blockType]++;
680 // p->num_BlockBytes_forType[blockType] += size;
681 if (size > blockSizeAllowedMax
682 || blockType == kBlockType_Reserved)
683 {
684 _unsupportedBlock = true;
685 if (ZstdDecInfo_GET_NUM_FRAMES(p) == 1 && p->num_Blocks == 1)
686 return S_FALSE;
687 break;
688 }
689 if (blockType == kBlockType_RLE)
690 size = 1;
691 _phySize = sb.GetCurOffset() + size;
692 RINOK(sb.Skip(size))
693 if (b & 1)
694 {
695 // it's last block
696 blocksWereParsed = true;
697 break;
698 }
699 }
700
701 if (!blocksWereParsed)
702 break;
703
704 if (fh.Is_Checksum())
705 {
706 _phySize = sb.GetCurOffset() + 4;
707 RINOK(sb.Read(4))
708 if (sb.Avail() < 4)
709 break;
710 p->checksum_Defined = true;
711 // if (p->num_DataFrames == 1)
712 p->checksum = GetUi32(sb.GetPtr());
713 sb.SkipInBuf(4);
714 }
715 }
716
717 if (ZstdDecInfo_GET_NUM_FRAMES(p) == 0)
718 return S_FALSE;
719
720 _needSeekToStart = true;
721 // } // _parseMode
722 _isArc = true;
723 _stream = stream;
724 _seqStream = stream;
725
726 return S_OK;
727 COM_TRY_END
728 }
729
730
Z7_COM7F_IMF(CHandler::OpenSeq (ISequentialInStream * stream))731 Z7_COM7F_IMF(CHandler::OpenSeq(ISequentialInStream *stream))
732 {
733 Close();
734 _isArc = true;
735 _seqStream = stream;
736 return S_OK;
737 }
738
739
Z7_COM7F_IMF(CHandler::Close ())740 Z7_COM7F_IMF(CHandler::Close())
741 {
742 _isArc = false;
743 _needSeekToStart = false;
744 // _dataAfterEnd = false;
745 // _needMoreInput = false;
746 _unsupportedBlock = false;
747
748 _wasParsed = false;
749 _phySize_Decoded_Defined = false;
750 _unpackSize_Defined = false;
751 _decoded_Info_Defined = false;
752
753 ZstdDecInfo_CLEAR(&_parsed_Info)
754 ZstdDecInfo_CLEAR(&_decoded_Info)
755
756 _phySize = 0;
757 _phySize_Decoded = 0;
758 _unpackSize = 0;
759
760 _seqStream.Release();
761 _stream.Release();
762 return S_OK;
763 }
764
765
Z7_COM7F_IMF(CHandler::Extract (const UInt32 * indices,UInt32 numItems,Int32 testMode,IArchiveExtractCallback * extractCallback))766 Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems,
767 Int32 testMode, IArchiveExtractCallback *extractCallback))
768 {
769 COM_TRY_BEGIN
770 if (numItems == 0)
771 return S_OK;
772 if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0))
773 return E_INVALIDARG;
774 if (_wasParsed)
775 {
776 RINOK(extractCallback->SetTotal(_phySize))
777 }
778
779 Int32 opRes;
780 {
781 CMyComPtr<ISequentialOutStream> realOutStream;
782 const Int32 askMode = testMode ?
783 NExtract::NAskMode::kTest :
784 NExtract::NAskMode::kExtract;
785 RINOK(extractCallback->GetStream(0, &realOutStream, askMode))
786 if (!testMode && !realOutStream)
787 return S_OK;
788
789 extractCallback->PrepareOperation(askMode);
790
791 if (_needSeekToStart)
792 {
793 if (!_stream)
794 return E_FAIL;
795 RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL))
796 }
797 else
798 _needSeekToStart = true;
799
800 CMyComPtr2_Create<ICompressProgressInfo, CLocalProgress> lps;
801 lps->Init(extractCallback, true);
802
803 #ifdef Z7_USE_ZSTD_ORIG_DECODER
804 CMyComPtr2_Create<ICompressCoder, NCompress::NZstd2::CDecoder> decoder;
805 #else
806 CMyComPtr2_Create<ICompressCoder, NCompress::NZstd::CDecoder> decoder;
807 #endif
808
809 CMyComPtr2_Create<ISequentialOutStream, CDummyOutStream> outStreamSpec;
810 outStreamSpec->SetStream(realOutStream);
811 outStreamSpec->Init();
812 // realOutStream.Release();
813
814 decoder->FinishMode = true;
815 #ifndef Z7_USE_ZSTD_ORIG_DECODER
816 decoder->DisableHash = _disableHash;
817 #endif
818
819 // _dataAfterEnd = false;
820 // _needMoreInput = false;
821 const HRESULT hres = decoder.Interface()->Code(_seqStream, outStreamSpec, NULL, NULL, lps);
822 /*
823 {
824 UInt64 t1 = decoder->GetInputProcessedSize();
825 // for debug
826 const UInt32 kTempSize = 64;
827 Byte buf[kTempSize];
828 UInt32 processedSize = 0;
829 RINOK(decoder->ReadUnusedFromInBuf(buf, kTempSize, &processedSize))
830 processedSize -= processedSize;
831 UInt64 t2 = decoder->GetInputProcessedSize();
832 t2 = t2;
833 t1 = t1;
834 }
835 */
836 const UInt64 outSize = outStreamSpec->GetSize();
837 // }
838
839 // if (hres == E_ABORT) return hres;
840 opRes = NExtract::NOperationResult::kDataError;
841
842 if (hres == E_OUTOFMEMORY)
843 {
844 return hres;
845 // opRes = NExtract::NOperationResult::kMemError;
846 }
847 else if (hres == S_OK || hres == S_FALSE)
848 {
849 #ifndef Z7_USE_ZSTD_ORIG_DECODER
850 _decoded_Info_Defined = true;
851 _decoded_Info = decoder->_state.info;
852 // NumDataFrames_Decoded = decoder->_state.info.num_DataFrames;
853 // NumSkipFrames_Decoded = decoder->_state.info.num_SkipFrames;
854 const UInt64 inSize = decoder->_inProcessed;
855 #else
856 const UInt64 inSize = decoder->GetInputProcessedSize();
857 #endif
858 _phySize_Decoded = inSize;
859 _phySize_Decoded_Defined = true;
860
861 _unpackSize_Defined = true;
862 _unpackSize = outSize;
863
864 // RINOK(
865 lps.Interface()->SetRatioInfo(&inSize, &outSize);
866
867 #ifdef Z7_USE_ZSTD_ORIG_DECODER
868 if (hres == S_OK)
869 opRes = NExtract::NOperationResult::kOK;
870 #else
871 if (decoder->ResInfo.decode_SRes == SZ_ERROR_CRC)
872 {
873 opRes = NExtract::NOperationResult::kCRCError;
874 }
875 else if (decoder->ResInfo.decode_SRes == SZ_ERROR_NO_ARCHIVE)
876 {
877 _isArc = false;
878 opRes = NExtract::NOperationResult::kIsNotArc;
879 }
880 else if (decoder->ResInfo.decode_SRes == SZ_ERROR_INPUT_EOF)
881 opRes = NExtract::NOperationResult::kUnexpectedEnd;
882 else
883 {
884 if (hres == S_OK && decoder->ResInfo.decode_SRes == SZ_OK)
885 opRes = NExtract::NOperationResult::kOK;
886 if (decoder->ResInfo.extraSize)
887 {
888 // if (inSize == 0) _isArc = false;
889 opRes = NExtract::NOperationResult::kDataAfterEnd;
890 }
891 /*
892 if (decoder->ResInfo.unexpededEnd)
893 opRes = NExtract::NOperationResult::kUnexpectedEnd;
894 */
895 }
896 #endif
897 }
898 else if (hres == E_NOTIMPL)
899 {
900 opRes = NExtract::NOperationResult::kUnsupportedMethod;
901 }
902 else
903 return hres;
904 }
905
906 return extractCallback->SetOperationResult(opRes);
907
908 COM_TRY_END
909 }
910
911
912
Z7_COM7F_IMF(CHandler::SetProperties (const wchar_t * const * names,const PROPVARIANT * values,UInt32 numProps))913 Z7_COM7F_IMF(CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps))
914 {
915 // return _props.SetProperties(names, values, numProps);
916 // _smallMode = false;
917 _disableHash = false;
918 _parseMode = false;
919 // _parseMode = true; // for debug
920 #ifdef Z7_USE_ZSTD_COMPRESSION
921 _props.Init();
922 #endif
923
924 for (UInt32 i = 0; i < numProps; i++)
925 {
926 UString name = names[i];
927 const PROPVARIANT &value = values[i];
928
929 if (name.IsEqualTo("parse"))
930 {
931 bool parseMode = true;
932 RINOK(PROPVARIANT_to_bool(value, parseMode))
933 _parseMode = parseMode;
934 continue;
935 }
936 if (name.IsPrefixedBy_Ascii_NoCase("crc"))
937 {
938 name.Delete(0, 3);
939 UInt32 crcSize = 4;
940 RINOK(ParsePropToUInt32(name, value, crcSize))
941 if (crcSize == 0)
942 _disableHash = true;
943 else if (crcSize == 4)
944 _disableHash = false;
945 else
946 return E_INVALIDARG;
947 continue;
948 }
949 #ifdef Z7_USE_ZSTD_COMPRESSION
950 /*
951 if (name.IsEqualTo("small"))
952 {
953 bool smallMode = true;
954 RINOK(PROPVARIANT_to_bool(value, smallMode))
955 _smallMode = smallMode;
956 continue;
957 }
958 */
959 RINOK(_props.SetProperty(names[i], value))
960 #endif
961 }
962 return S_OK;
963 }
964
965
966
967
968 #ifdef Z7_USE_ZSTD_COMPRESSION
969
Z7_COM7F_IMF(CHandler::GetFileTimeType (UInt32 * timeType))970 Z7_COM7F_IMF(CHandler::GetFileTimeType(UInt32 *timeType))
971 {
972 *timeType = GET_FileTimeType_NotDefined_for_GetFileTimeType;
973 // *timeType = NFileTimeType::kUnix;
974 return S_OK;
975 }
976
977
Z7_COM7F_IMF(CHandler::UpdateItems (ISequentialOutStream * outStream,UInt32 numItems,IArchiveUpdateCallback * updateCallback))978 Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems,
979 IArchiveUpdateCallback *updateCallback))
980 {
981 COM_TRY_BEGIN
982
983 if (numItems != 1)
984 return E_INVALIDARG;
985 {
986 CMyComPtr<IStreamSetRestriction> setRestriction;
987 outStream->QueryInterface(IID_IStreamSetRestriction, (void **)&setRestriction);
988 if (setRestriction)
989 RINOK(setRestriction->SetRestriction(0, 0))
990 }
991 Int32 newData, newProps;
992 UInt32 indexInArchive;
993 if (!updateCallback)
994 return E_FAIL;
995 RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive))
996
997 if (IntToBool(newProps))
998 {
999 {
1000 NCOM::CPropVariant prop;
1001 RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop))
1002 if (prop.vt != VT_EMPTY)
1003 if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE)
1004 return E_INVALIDARG;
1005 }
1006 }
1007
1008 if (IntToBool(newData))
1009 {
1010 UInt64 size;
1011 {
1012 NCOM::CPropVariant prop;
1013 RINOK(updateCallback->GetProperty(0, kpidSize, &prop))
1014 if (prop.vt != VT_UI8)
1015 return E_INVALIDARG;
1016 size = prop.uhVal.QuadPart;
1017 }
1018
1019 if (!_props.MethodName.IsEmpty()
1020 && !_props.MethodName.IsEqualTo_Ascii_NoCase("zstd"))
1021 return E_INVALIDARG;
1022
1023 {
1024 CMyComPtr<ISequentialInStream> fileInStream;
1025 RINOK(updateCallback->GetStream(0, &fileInStream))
1026 if (!fileInStream)
1027 return S_FALSE;
1028 {
1029 CMyComPtr<IStreamGetSize> streamGetSize;
1030 fileInStream.QueryInterface(IID_IStreamGetSize, &streamGetSize);
1031 if (streamGetSize)
1032 {
1033 UInt64 size2;
1034 if (streamGetSize->GetSize(&size2) == S_OK)
1035 size = size2;
1036 }
1037 }
1038 RINOK(updateCallback->SetTotal(size))
1039
1040 CMethodProps props2 = _props;
1041
1042 #ifndef Z7_ST
1043 /*
1044 CSingleMethodProps (_props)
1045 derives from
1046 CMethodProps (props2)
1047 So we transfer additional variable (num Threads) to CMethodProps list of properties
1048 */
1049
1050 UInt32 numThreads = _props._numThreads;
1051
1052 if (numThreads > Z7_ZSTDMT_NBWORKERS_MAX)
1053 numThreads = Z7_ZSTDMT_NBWORKERS_MAX;
1054
1055 if (_props.FindProp(NCoderPropID::kNumThreads) < 0)
1056 {
1057 if (!_props._numThreads_WasForced
1058 && numThreads >= 1
1059 && _props._memUsage_WasSet)
1060 {
1061 NCompress::NZstd::CEncoderProps zstdProps;
1062 RINOK(zstdProps.SetFromMethodProps(_props))
1063 ZstdEncProps_NormalizeFull(&zstdProps.EncProps);
1064 numThreads = ZstdEncProps_GetNumThreads_for_MemUsageLimit(
1065 &zstdProps.EncProps, _props._memUsage_Compress, numThreads);
1066 }
1067 props2.AddProp_NumThreads(numThreads);
1068 }
1069
1070 #endif // Z7_ST
1071
1072 CMyComPtr2_Create<ICompressProgressInfo, CLocalProgress> lps;
1073 lps->Init(updateCallback, true);
1074 {
1075 CMyComPtr2_Create<ICompressCoder, NCompress::NZstd::CEncoder> encoder;
1076 // size = 1 << 24; // for debug
1077 RINOK(props2.SetCoderProps(encoder.ClsPtr(), size != (UInt64)(Int64)-1 ? &size : NULL))
1078 // encoderSpec->_props.SmallFileOpt = _smallMode;
1079 // we must set kExpectedDataSize just before Code().
1080 encoder->SrcSizeHint64 = size;
1081 /*
1082 CMyComPtr<ICompressSetCoderPropertiesOpt> optProps;
1083 _compressEncoder->QueryInterface(IID_ICompressSetCoderPropertiesOpt, (void**)&optProps);
1084 if (optProps)
1085 {
1086 PROPID propID = NCoderPropID::kExpectedDataSize;
1087 NWindows::NCOM::CPropVariant prop = (UInt64)size;
1088 // RINOK(optProps->SetCoderPropertiesOpt(&propID, &prop, 1))
1089 RINOK(encoderSpec->SetCoderPropertiesOpt(&propID, &prop, 1))
1090 }
1091 */
1092 RINOK(encoder.Interface()->Code(fileInStream, outStream, NULL, NULL, lps))
1093 }
1094 }
1095 return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK);
1096 }
1097
1098 if (indexInArchive != 0)
1099 return E_INVALIDARG;
1100
1101 CMyComPtr2_Create<ICompressProgressInfo, CLocalProgress> lps;
1102 lps->Init(updateCallback, true);
1103
1104 CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
1105 updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
1106 if (opCallback)
1107 {
1108 RINOK(opCallback->ReportOperation(
1109 NEventIndexType::kInArcIndex, 0,
1110 NUpdateNotifyOp::kReplicate))
1111 }
1112
1113 if (_stream)
1114 RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL))
1115
1116 return NCompress::CopyStream(_stream, outStream, lps);
1117
1118 COM_TRY_END
1119 }
1120 #endif
1121
1122
1123
1124 #ifndef Z7_USE_ZSTD_COMPRESSION
1125 #undef IMP_CreateArcOut
1126 #define IMP_CreateArcOut
1127 #undef CreateArcOut
1128 #define CreateArcOut NULL
1129 #endif
1130
1131 #ifdef Z7_USE_ZSTD_COMPRESSION
1132 REGISTER_ARC_IO(
1133 "zstd2", "zst tzst", "* .tar", 0xe + 1,
1134 k_Signature, 0
1135 , NArcInfoFlags::kKeepName
1136 , 0
1137 , NULL)
1138 #else
1139 REGISTER_ARC_IO(
1140 "zstd", "zst tzst", "* .tar", 0xe,
1141 k_Signature, 0
1142 , NArcInfoFlags::kKeepName
1143 , 0
1144 , NULL)
1145 #endif
1146
1147 }}
1148