1 // ZipUpdate.cpp
2
3 #include "StdAfx.h"
4
5 // #define DEBUG_CACHE
6
7 #ifdef DEBUG_CACHE
8 #include <stdio.h>
9 #define PRF(x) x
10 #else
11 #define PRF(x)
12 #endif
13
14 #include "../../../../C/Alloc.h"
15
16 #include "../../../Common/AutoPtr.h"
17 #include "../../../Common/Defs.h"
18 #include "../../../Common/StringConvert.h"
19
20 #include "../../../Windows/TimeUtils.h"
21 #include "../../../Windows/Thread.h"
22
23 #include "../../Common/CreateCoder.h"
24 #include "../../Common/LimitedStreams.h"
25 #include "../../Common/OutMemStream.h"
26 #include "../../Common/ProgressUtils.h"
27 #ifndef Z7_ST
28 #include "../../Common/ProgressMt.h"
29 #endif
30 #include "../../Common/StreamUtils.h"
31
32 #include "../../Compress/CopyCoder.h"
33 // #include "../../Compress/ZstdEncoderProps.h"
34
35 #include "ZipAddCommon.h"
36 #include "ZipOut.h"
37 #include "ZipUpdate.h"
38
39 using namespace NWindows;
40 using namespace NSynchronization;
41
42 namespace NArchive {
43 namespace NZip {
44
45 static const Byte kHostOS =
46 #ifdef _WIN32
47 NFileHeader::NHostOS::kFAT;
48 #else
49 NFileHeader::NHostOS::kUnix;
50 #endif
51
52 static const Byte kMadeByHostOS = kHostOS;
53
54 // 18.06: now we always write zero to high byte of ExtractVersion field.
55 // Previous versions of p7zip wrote (NFileHeader::NHostOS::kUnix) there, that is not correct
56 static const Byte kExtractHostOS = 0;
57
58 static const Byte kMethodForDirectory = NFileHeader::NCompressionMethod::kStore;
59
60
AddAesExtra(CItem & item,Byte aesKeyMode,UInt16 method)61 static void AddAesExtra(CItem &item, Byte aesKeyMode, UInt16 method)
62 {
63 CWzAesExtra wzAesField;
64 wzAesField.Strength = aesKeyMode;
65 wzAesField.Method = method;
66 item.Method = NFileHeader::NCompressionMethod::kWzAES;
67 item.Crc = 0;
68 CExtraSubBlock sb;
69 wzAesField.SetSubBlock(sb);
70 item.LocalExtra.SubBlocks.Add(sb);
71 item.CentralExtra.SubBlocks.Add(sb);
72 }
73
74
Copy_From_UpdateItem_To_ItemOut(const CUpdateItem & ui,CItemOut & item)75 static void Copy_From_UpdateItem_To_ItemOut(const CUpdateItem &ui, CItemOut &item)
76 {
77 item.Name = ui.Name;
78 item.Name_Utf = ui.Name_Utf;
79 item.Comment = ui.Comment;
80 item.SetUtf8(ui.IsUtf8);
81 // item.SetFlag_AltStream(ui.IsAltStream);
82 // item.ExternalAttrib = ui.Attrib;
83 item.Time = ui.Time;
84 item.Ntfs_MTime = ui.Ntfs_MTime;
85 item.Ntfs_ATime = ui.Ntfs_ATime;
86 item.Ntfs_CTime = ui.Ntfs_CTime;
87
88 item.Write_UnixTime = ui.Write_UnixTime;
89 item.Write_NtfsTime = ui.Write_NtfsTime;
90 }
91
SetFileHeader(const CCompressionMethodMode & options,const CUpdateItem & ui,bool useDescriptor,CItemOut & item)92 static void SetFileHeader(
93 const CCompressionMethodMode &options,
94 const CUpdateItem &ui,
95 bool useDescriptor,
96 CItemOut &item)
97 {
98 item.Size = ui.Size;
99 const bool isDir = ui.IsDir;
100
101 item.ClearFlags();
102
103 if (ui.NewProps)
104 {
105 Copy_From_UpdateItem_To_ItemOut(ui, item);
106 // item.SetFlag_AltStream(ui.IsAltStream);
107 item.ExternalAttrib = ui.Attrib;
108 }
109 /*
110 else
111 isDir = item.IsDir();
112 */
113
114 item.MadeByVersion.HostOS = kMadeByHostOS;
115 item.MadeByVersion.Version = NFileHeader::NCompressionMethod::kMadeByProgramVersion;
116
117 item.ExtractVersion.HostOS = kExtractHostOS;
118
119 item.InternalAttrib = 0; // test it
120 item.SetEncrypted(!isDir && options.Password_Defined);
121 item.SetDescriptorMode(useDescriptor);
122
123 if (isDir)
124 {
125 item.ExtractVersion.Version = NFileHeader::NCompressionMethod::kExtractVersion_Dir;
126 item.Method = kMethodForDirectory;
127 item.PackSize = 0;
128 item.Size = 0;
129 item.Crc = 0;
130 }
131
132 item.LocalExtra.Clear();
133 item.CentralExtra.Clear();
134
135 if (isDir)
136 {
137 item.ExtractVersion.Version = NFileHeader::NCompressionMethod::kExtractVersion_Dir;
138 item.Method = kMethodForDirectory;
139 item.PackSize = 0;
140 item.Size = 0;
141 item.Crc = 0;
142 }
143 else if (options.IsRealAesMode())
144 AddAesExtra(item, options.AesKeyMode, (Byte)(options.MethodSequence.IsEmpty() ? 8 : options.MethodSequence[0]));
145 }
146
147
148 // we call SetItemInfoFromCompressingResult() after SetFileHeader()
149
SetItemInfoFromCompressingResult(const CCompressingResult & compressingResult,bool isAesMode,Byte aesKeyMode,CItem & item)150 static void SetItemInfoFromCompressingResult(const CCompressingResult &compressingResult,
151 bool isAesMode, Byte aesKeyMode, CItem &item)
152 {
153 item.ExtractVersion.Version = compressingResult.ExtractVersion;
154 item.Method = compressingResult.Method;
155 if (compressingResult.Method == NFileHeader::NCompressionMethod::kLZMA && compressingResult.LzmaEos)
156 item.Flags |= NFileHeader::NFlags::kLzmaEOS;
157 item.Crc = compressingResult.CRC;
158 item.Size = compressingResult.UnpackSize;
159 item.PackSize = compressingResult.PackSize;
160
161 item.LocalExtra.Clear();
162 item.CentralExtra.Clear();
163
164 if (isAesMode)
165 AddAesExtra(item, aesKeyMode, compressingResult.Method);
166 }
167
168
169 #ifndef Z7_ST
170
171 struct CMtSem
172 {
173 NWindows::NSynchronization::CSemaphore Semaphore;
174 NWindows::NSynchronization::CCriticalSection CS;
175 CIntVector Indexes;
176 int Head;
177
ReleaseItemNArchive::NZip::CMtSem178 void ReleaseItem(unsigned index)
179 {
180 {
181 CCriticalSectionLock lock(CS);
182 Indexes[index] = Head;
183 Head = (int)index;
184 }
185 Semaphore.Release();
186 }
187
GetFreeItemNArchive::NZip::CMtSem188 int GetFreeItem()
189 {
190 int i;
191 {
192 CCriticalSectionLock lock(CS);
193 i = Head;
194 Head = Indexes[(unsigned)i];
195 }
196 return i;
197 }
198 };
199
200 static THREAD_FUNC_DECL CoderThread(void *threadCoderInfo);
201
202 struct CThreadInfo
203 {
204 DECL_EXTERNAL_CODECS_LOC_VARS_DECL
205
206 NWindows::CThread Thread;
207 NWindows::NSynchronization::CAutoResetEvent CompressEvent;
208 CMtSem *MtSem;
209 unsigned ThreadIndex;
210
211 bool ExitThread;
212
213 CMtCompressProgress *ProgressSpec;
214 CMyComPtr<ICompressProgressInfo> Progress;
215
216 COutMemStream *OutStreamSpec;
217 CMyComPtr<IOutStream> OutStream;
218 CMyComPtr<ISequentialInStream> InStream;
219
220 CAddCommon Coder;
221 HRESULT Result;
222 CCompressingResult CompressingResult;
223
224 bool IsFree;
225 bool InSeqMode;
226 bool OutSeqMode;
227 bool ExpectedDataSize_IsConfirmed;
228
229 UInt32 UpdateIndex;
230 UInt32 FileTime;
231 UInt64 ExpectedDataSize;
232
CThreadInfoNArchive::NZip::CThreadInfo233 CThreadInfo():
234 MtSem(NULL),
235 ExitThread(false),
236 ProgressSpec(NULL),
237 OutStreamSpec(NULL),
238 IsFree(true),
239 InSeqMode(false),
240 OutSeqMode(false),
241 ExpectedDataSize_IsConfirmed(false),
242 FileTime(0),
243 ExpectedDataSize((UInt64)(Int64)-1)
244 {}
245
SetOptionsNArchive::NZip::CThreadInfo246 void SetOptions(const CCompressionMethodMode &options)
247 {
248 Coder.SetOptions(options);
249 }
250
CreateEventsNArchive::NZip::CThreadInfo251 HRESULT CreateEvents()
252 {
253 WRes wres = CompressEvent.CreateIfNotCreated_Reset();
254 return HRESULT_FROM_WIN32(wres);
255 }
256
CreateThreadNArchive::NZip::CThreadInfo257 HRESULT CreateThread()
258 {
259 WRes wres = Thread.Create(CoderThread, this);
260 return HRESULT_FROM_WIN32(wres);
261 }
262
263 void WaitAndCode();
264
StopWait_CloseNArchive::NZip::CThreadInfo265 void StopWait_Close()
266 {
267 ExitThread = true;
268 if (OutStreamSpec)
269 OutStreamSpec->StopWriting(E_ABORT);
270 if (CompressEvent.IsCreated())
271 CompressEvent.Set();
272 Thread.Wait_Close();
273 }
274 };
275
WaitAndCode()276 void CThreadInfo::WaitAndCode()
277 {
278 for (;;)
279 {
280 CompressEvent.Lock();
281 if (ExitThread)
282 return;
283
284 Result = Coder.Compress(
285 EXTERNAL_CODECS_LOC_VARS
286 InStream, OutStream,
287 InSeqMode, OutSeqMode, FileTime, ExpectedDataSize,
288 ExpectedDataSize_IsConfirmed,
289 Progress, CompressingResult);
290
291 if (Result == S_OK && Progress)
292 Result = Progress->SetRatioInfo(&CompressingResult.UnpackSize, &CompressingResult.PackSize);
293
294 MtSem->ReleaseItem(ThreadIndex);
295 }
296 }
297
CoderThread(void * threadCoderInfo)298 static THREAD_FUNC_DECL CoderThread(void *threadCoderInfo)
299 {
300 ((CThreadInfo *)threadCoderInfo)->WaitAndCode();
301 return 0;
302 }
303
304 class CThreads
305 {
306 public:
307 CObjectVector<CThreadInfo> Threads;
~CThreads()308 ~CThreads()
309 {
310 FOR_VECTOR (i, Threads)
311 Threads[i].StopWait_Close();
312 }
313 };
314
315 struct CMemBlocks2: public CMemLockBlocks
316 {
317 bool Skip;
318 bool InSeqMode;
319 bool PreDescriptorMode;
320 bool Finished;
321 CCompressingResult CompressingResult;
322
CMemBlocks2NArchive::NZip::CMemBlocks2323 CMemBlocks2(): Skip(false), InSeqMode(false), PreDescriptorMode(false), Finished(false),
324 CompressingResult() {}
325 };
326
327 class CMemRefs
328 {
329 public:
330 CMemBlockManagerMt *Manager;
331 CObjectVector<CMemBlocks2> Refs;
CMemRefs(CMemBlockManagerMt * manager)332 CMemRefs(CMemBlockManagerMt *manager): Manager(manager) {}
~CMemRefs()333 ~CMemRefs()
334 {
335 FOR_VECTOR (i, Refs)
336 Refs[i].FreeOpt(Manager);
337 }
338 };
339
340
341 Z7_CLASS_IMP_NOQIB_1(
342 CMtProgressMixer2
343 , ICompressProgressInfo
344 )
345 UInt64 ProgressOffset;
346 UInt64 InSizes[2];
347 UInt64 OutSizes[2];
348 CMyComPtr<IProgress> Progress;
349 CMyComPtr<ICompressProgressInfo> RatioProgress;
350 bool _inSizeIsMain;
351 public:
352 NWindows::NSynchronization::CCriticalSection CriticalSection;
353 void Create(IProgress *progress, bool inSizeIsMain);
354 void SetProgressOffset(UInt64 progressOffset);
355 void SetProgressOffset_NoLock(UInt64 progressOffset);
356 HRESULT SetRatioInfo(unsigned index, const UInt64 *inSize, const UInt64 *outSize);
357 };
358
359 void CMtProgressMixer2::Create(IProgress *progress, bool inSizeIsMain)
360 {
361 Progress = progress;
362 Progress.QueryInterface(IID_ICompressProgressInfo, &RatioProgress);
363 _inSizeIsMain = inSizeIsMain;
364 ProgressOffset = InSizes[0] = InSizes[1] = OutSizes[0] = OutSizes[1] = 0;
365 }
366
367 void CMtProgressMixer2::SetProgressOffset_NoLock(UInt64 progressOffset)
368 {
369 InSizes[1] = OutSizes[1] = 0;
370 ProgressOffset = progressOffset;
371 }
372
373 void CMtProgressMixer2::SetProgressOffset(UInt64 progressOffset)
374 {
375 CriticalSection.Enter();
376 SetProgressOffset_NoLock(progressOffset);
377 CriticalSection.Leave();
378 }
379
380 HRESULT CMtProgressMixer2::SetRatioInfo(unsigned index, const UInt64 *inSize, const UInt64 *outSize)
381 {
382 NWindows::NSynchronization::CCriticalSectionLock lock(CriticalSection);
383 if (index == 0 && RatioProgress)
384 {
385 RINOK(RatioProgress->SetRatioInfo(inSize, outSize))
386 }
387 if (inSize)
388 InSizes[index] = *inSize;
389 if (outSize)
390 OutSizes[index] = *outSize;
391 UInt64 v = ProgressOffset + (_inSizeIsMain ?
392 (InSizes[0] + InSizes[1]) :
393 (OutSizes[0] + OutSizes[1]));
394 return Progress->SetCompleted(&v);
395 }
396
397 Z7_COM7F_IMF(CMtProgressMixer2::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize))
398 {
399 return SetRatioInfo(0, inSize, outSize);
400 }
401
402
403 Z7_CLASS_IMP_NOQIB_1(
404 CMtProgressMixer
405 , ICompressProgressInfo
406 )
407 public:
408 CMtProgressMixer2 *Mixer2;
409 CMyComPtr<ICompressProgressInfo> RatioProgress;
410 void Create(IProgress *progress, bool inSizeIsMain);
411 };
412
413 void CMtProgressMixer::Create(IProgress *progress, bool inSizeIsMain)
414 {
415 Mixer2 = new CMtProgressMixer2;
416 RatioProgress = Mixer2;
417 Mixer2->Create(progress, inSizeIsMain);
418 }
419
420 Z7_COM7F_IMF(CMtProgressMixer::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize))
421 {
422 return Mixer2->SetRatioInfo(1, inSize, outSize);
423 }
424
425
426 #endif
427
428 static HRESULT UpdateItemOldData(
429 COutArchive &archive,
430 CInArchive *inArchive,
431 const CItemEx &itemEx,
432 const CUpdateItem &ui,
433 CItemOut &item,
434 /* bool izZip64, */
435 ICompressProgressInfo *progress,
436 IArchiveUpdateCallbackFile *opCallback,
437 UInt64 &complexity)
438 {
439 if (opCallback)
440 {
441 RINOK(opCallback->ReportOperation(
442 NEventIndexType::kInArcIndex, (UInt32)ui.IndexInArc,
443 NUpdateNotifyOp::kReplicate))
444 }
445
446 UInt64 rangeSize;
447
448 RINOK(archive.ClearRestriction())
449
450 if (ui.NewProps)
451 {
452 if (item.HasDescriptor())
453 return E_NOTIMPL;
454
455 // we keep ExternalAttrib and some another properties from old archive
456 // item.ExternalAttrib = ui.Attrib;
457 // if we don't change Comment, we keep Comment from OldProperties
458 Copy_From_UpdateItem_To_ItemOut(ui, item);
459 // item.SetFlag_AltStream(ui.IsAltStream);
460
461 item.CentralExtra.RemoveUnknownSubBlocks();
462 item.LocalExtra.RemoveUnknownSubBlocks();
463
464 archive.WriteLocalHeader(item);
465 rangeSize = item.GetPackSizeWithDescriptor();
466 }
467 else
468 {
469 item.LocalHeaderPos = archive.GetCurPos();
470 rangeSize = itemEx.GetLocalFullSize();
471 }
472
473 CMyComPtr<ISequentialInStream> packStream;
474
475 RINOK(inArchive->GetItemStream(itemEx, ui.NewProps, packStream))
476 if (!packStream)
477 return E_NOTIMPL;
478
479 complexity += rangeSize;
480
481 CMyComPtr<ISequentialOutStream> outStream;
482 archive.CreateStreamForCopying(outStream);
483 HRESULT res = NCompress::CopyStream_ExactSize(packStream, outStream, rangeSize, progress);
484 archive.MoveCurPos(rangeSize);
485 return res;
486 }
487
488
489 static HRESULT WriteDirHeader(COutArchive &archive, const CCompressionMethodMode *options,
490 const CUpdateItem &ui, CItemOut &item)
491 {
492 SetFileHeader(*options, ui, false, item);
493 RINOK(archive.ClearRestriction())
494 archive.WriteLocalHeader(item);
495 return S_OK;
496 }
497
498
499 static void UpdatePropsFromStream(
500 const CUpdateOptions &options,
501 CUpdateItem &item, ISequentialInStream *fileInStream,
502 IArchiveUpdateCallback *updateCallback, UInt64 &totalComplexity)
503 {
504 CMyComPtr<IStreamGetProps> getProps;
505 fileInStream->QueryInterface(IID_IStreamGetProps, (void **)&getProps);
506 UInt64 size = (UInt64)(Int64)-1;
507 bool size_WasSet = false;
508
509 if (getProps)
510 {
511 FILETIME cTime, aTime, mTime;
512 UInt32 attrib;
513 if (getProps->GetProps(&size, &cTime, &aTime, &mTime, &attrib) == S_OK)
514 {
515 if (options.Write_MTime)
516 if (!FILETIME_IsZero(mTime))
517 {
518 item.Ntfs_MTime = mTime;
519 NTime::UtcFileTime_To_LocalDosTime(mTime, item.Time);
520 }
521
522 if (options.Write_CTime) if (!FILETIME_IsZero(cTime)) item.Ntfs_CTime = cTime;
523 if (options.Write_ATime) if (!FILETIME_IsZero(aTime)) item.Ntfs_ATime = aTime;
524
525 item.Attrib = attrib;
526 size_WasSet = true;
527 }
528 }
529
530 if (!size_WasSet)
531 {
532 CMyComPtr<IStreamGetSize> streamGetSize;
533 fileInStream->QueryInterface(IID_IStreamGetSize, (void **)&streamGetSize);
534 if (streamGetSize)
535 {
536 if (streamGetSize->GetSize(&size) == S_OK)
537 size_WasSet = true;
538 }
539 }
540
541 if (size_WasSet && size != (UInt64)(Int64)-1)
542 {
543 item.Size_WasSetFromStream = true;
544 if (size != item.Size)
545 {
546 const Int64 newComplexity = (Int64)totalComplexity + ((Int64)size - (Int64)item.Size);
547 if (newComplexity > 0)
548 {
549 totalComplexity = (UInt64)newComplexity;
550 updateCallback->SetTotal(totalComplexity);
551 }
552 item.Size = size;
553 }
554 }
555 }
556
557
558 /*
559 static HRESULT ReportProps(
560 IArchiveUpdateCallbackArcProp *reportArcProp,
561 UInt32 index,
562 const CItemOut &item,
563 bool isAesMode)
564 {
565 PROPVARIANT prop;
566 prop.vt = VT_EMPTY;
567 prop.wReserved1 = 0;
568
569 NCOM::PropVarEm_Set_UInt64(&prop, item.Size);
570 RINOK(reportArcProp->ReportProp(NEventIndexType::kOutArcIndex, index, kpidSize, &prop));
571
572 NCOM::PropVarEm_Set_UInt64(&prop, item.PackSize);
573 RINOK(reportArcProp->ReportProp(NEventIndexType::kOutArcIndex, index, kpidPackSize, &prop));
574
575 if (!isAesMode)
576 {
577 NCOM::PropVarEm_Set_UInt32(&prop, item.Crc);
578 RINOK(reportArcProp->ReportProp(NEventIndexType::kOutArcIndex, index, kpidCRC, &prop));
579 }
580
581 RINOK(reportArcProp->ReportFinished(NEventIndexType::kOutArcIndex, index, NUpdate::NOperationResult::kOK));
582
583 // if (opCallback) RINOK(opCallback->ReportOperation(NEventIndexType::kOutArcIndex, index, NUpdateNotifyOp::kOpFinished))
584
585 return S_OK;
586 }
587 */
588
589 /*
590 struct CTotalStats
591 {
592 UInt64 Size;
593 UInt64 PackSize;
594
595 void UpdateWithItem(const CItemOut &item)
596 {
597 Size += item.Size;
598 PackSize += item.PackSize;
599 }
600 };
601
602 static HRESULT ReportArcProps(IArchiveUpdateCallbackArcProp *reportArcProp,
603 CTotalStats &st)
604 {
605 PROPVARIANT prop;
606 prop.vt = VT_EMPTY;
607 prop.wReserved1 = 0;
608 {
609 NWindows::NCOM::PropVarEm_Set_UInt64(&prop, st.Size);
610 RINOK(reportArcProp->ReportProp(
611 NEventIndexType::kArcProp, 0, kpidSize, &prop));
612 }
613 {
614 NWindows::NCOM::PropVarEm_Set_UInt64(&prop, st.PackSize);
615 RINOK(reportArcProp->ReportProp(
616 NEventIndexType::kArcProp, 0, kpidPackSize, &prop));
617 }
618 return S_OK;
619 }
620 */
621
622
623 static HRESULT Update2St(
624 DECL_EXTERNAL_CODECS_LOC_VARS
625 COutArchive &archive,
626 CInArchive *inArchive,
627 const CObjectVector<CItemEx> &inputItems,
628 CObjectVector<CUpdateItem> &updateItems,
629 const CUpdateOptions &updateOptions,
630 const CCompressionMethodMode *options, bool outSeqMode,
631 const CByteBuffer *comment,
632 IArchiveUpdateCallback *updateCallback,
633 UInt64 &totalComplexity,
634 IArchiveUpdateCallbackFile *opCallback
635 // , IArchiveUpdateCallbackArcProp *reportArcProp
636 )
637 {
638 CLocalProgress *lps = new CLocalProgress;
639 CMyComPtr<ICompressProgressInfo> progress = lps;
640 lps->Init(updateCallback, true);
641
642 CAddCommon compressor;
643 compressor.SetOptions(*options);
644
645 CObjectVector<CItemOut> items;
646 UInt64 unpackSizeTotal = 0, packSizeTotal = 0;
647
648 FOR_VECTOR (itemIndex, updateItems)
649 {
650 lps->InSize = unpackSizeTotal;
651 lps->OutSize = packSizeTotal;
652 RINOK(lps->SetCur())
653 CUpdateItem &ui = updateItems[itemIndex];
654 CItemEx itemEx;
655 CItemOut item;
656
657 if (!ui.NewProps || !ui.NewData)
658 {
659 // Note: for (ui.NewProps && !ui.NewData) it copies Props from old archive,
660 // But we will rewrite all important properties later. But we can keep some properties like Comment
661 itemEx = inputItems[(unsigned)ui.IndexInArc];
662 if (inArchive->Read_LocalItem_After_CdItem_Full(itemEx) != S_OK)
663 return E_NOTIMPL;
664 (CItem &)item = itemEx;
665 }
666
667 if (ui.NewData)
668 {
669 // bool isDir = ((ui.NewProps) ? ui.IsDir : item.IsDir());
670 bool isDir = ui.IsDir;
671 if (isDir)
672 {
673 RINOK(WriteDirHeader(archive, options, ui, item))
674 }
675 else
676 {
677 CMyComPtr<ISequentialInStream> fileInStream;
678 {
679 HRESULT res = updateCallback->GetStream(ui.IndexInClient, &fileInStream);
680 if (res == S_FALSE)
681 {
682 lps->ProgressOffset += ui.Size;
683 RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK))
684 continue;
685 }
686 RINOK(res)
687 if (!fileInStream)
688 return E_INVALIDARG;
689
690 bool inSeqMode = false;
691 if (!inSeqMode)
692 {
693 CMyComPtr<IInStream> inStream2;
694 fileInStream->QueryInterface(IID_IInStream, (void **)&inStream2);
695 inSeqMode = (inStream2 == NULL);
696 }
697 // seqMode = true; // to test seqMode
698
699 UpdatePropsFromStream(updateOptions, ui, fileInStream, updateCallback, totalComplexity);
700
701 CCompressingResult compressingResult;
702
703 RINOK(compressor.Set_Pre_CompressionResult(
704 inSeqMode, outSeqMode,
705 ui.Size,
706 compressingResult))
707
708 SetFileHeader(*options, ui, compressingResult.DescriptorMode, item);
709
710 // file Size can be 64-bit !!!
711
712 SetItemInfoFromCompressingResult(compressingResult, options->IsRealAesMode(), options->AesKeyMode, item);
713
714 RINOK(archive.SetRestrictionFromCurrent())
715 archive.WriteLocalHeader(item);
716
717 CMyComPtr<IOutStream> outStream;
718 archive.CreateStreamForCompressing(outStream);
719
720 RINOK(compressor.Compress(
721 EXTERNAL_CODECS_LOC_VARS
722 fileInStream, outStream,
723 inSeqMode, outSeqMode,
724 ui.Time,
725 ui.Size, ui.Size_WasSetFromStream,
726 progress, compressingResult))
727
728 if (item.HasDescriptor() != compressingResult.DescriptorMode)
729 return E_FAIL;
730
731 SetItemInfoFromCompressingResult(compressingResult, options->IsRealAesMode(), options->AesKeyMode, item);
732
733 archive.WriteLocalHeader_Replace(item);
734 }
735 // if (reportArcProp) RINOK(ReportProps(reportArcProp, ui.IndexInClient, item, options->IsRealAesMode()))
736 RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK))
737 unpackSizeTotal += item.Size;
738 packSizeTotal += item.PackSize;
739 }
740 }
741 else
742 {
743 UInt64 complexity = 0;
744 lps->SendRatio = false;
745
746 RINOK(UpdateItemOldData(archive, inArchive, itemEx, ui, item, progress, opCallback, complexity))
747
748 lps->SendRatio = true;
749 lps->ProgressOffset += complexity;
750 }
751
752 items.Add(item);
753 lps->ProgressOffset += kLocalHeaderSize;
754 }
755
756 lps->InSize = unpackSizeTotal;
757 lps->OutSize = packSizeTotal;
758 RINOK(lps->SetCur())
759
760 RINOK(archive.WriteCentralDir(items, comment))
761
762 /*
763 CTotalStats stat;
764 stat.Size = unpackSizeTotal;
765 stat.PackSize = packSizeTotal;
766 if (reportArcProp)
767 RINOK(ReportArcProps(reportArcProp, stat))
768 */
769
770 lps->ProgressOffset += kCentralHeaderSize * updateItems.Size() + 1;
771 return lps->SetCur();
772 }
773
774 #ifndef Z7_ST
775
776
777 static const size_t kBlockSize = 1 << 16;
778 // kMemPerThread must be >= kBlockSize
779 //
780 static const size_t kMemPerThread = (size_t)sizeof(size_t) << 23;
781 // static const size_t kMemPerThread = (size_t)sizeof(size_t) << 16; // for debug
782 // static const size_t kMemPerThread = (size_t)1 << 16; // for debug
783
784 /*
785 in:
786 nt_Zip >= 1: the starting maximum number of ZIP threads for search
787 out:
788 nt_Zip: calculated number of ZIP threads
789 returns: calculated number of ZSTD threads
790 */
791 /*
792 static UInt32 CalcThreads_for_ZipZstd(CZstdEncProps *zstdProps,
793 UInt64 memLimit, UInt32 totalThreads,
794 UInt32 &nt_Zip)
795 {
796 for (; nt_Zip > 1; nt_Zip--)
797 {
798 UInt64 mem1 = memLimit / nt_Zip;
799 if (mem1 <= kMemPerThread)
800 continue;
801 mem1 -= kMemPerThread;
802 UInt32 n_ZSTD = ZstdEncProps_GetNumThreads_for_MemUsageLimit(
803 zstdProps, mem1, totalThreads / nt_Zip);
804 // we don't allow (nbWorkers == 1) here
805 if (n_ZSTD <= 1)
806 n_ZSTD = 0;
807 zstdProps->nbWorkers = n_ZSTD;
808 mem1 = ZstdEncProps_GetMemUsage(zstdProps);
809 if ((mem1 + kMemPerThread) * nt_Zip <= memLimit)
810 return n_ZSTD;
811 }
812 return ZstdEncProps_GetNumThreads_for_MemUsageLimit(
813 zstdProps, memLimit, totalThreads);
814 }
815
816
817 static UInt32 SetZstdThreads(
818 const CCompressionMethodMode &options,
819 COneMethodInfo *oneMethodMain,
820 UInt32 numThreads,
821 UInt32 numZipThreads_limit,
822 UInt64 numFilesToCompress,
823 UInt64 numBytesToCompress)
824 {
825 NCompress::NZstd::CEncoderProps encoderProps;
826 RINOK(encoderProps.SetFromMethodProps(*oneMethodMain));
827 CZstdEncProps &zstdProps = encoderProps.EncProps;
828 ZstdEncProps_NormalizeFull(&zstdProps);
829 if (oneMethodMain->FindProp(NCoderPropID::kNumThreads) >= 0)
830 {
831 // threads for ZSTD are fixed
832 if (zstdProps.nbWorkers > 1)
833 numThreads /= zstdProps.nbWorkers;
834 if (numThreads > numZipThreads_limit)
835 numThreads = numZipThreads_limit;
836 if (options._memUsage_WasSet
837 && !options._numThreads_WasForced)
838 {
839 const UInt64 mem1 = ZstdEncProps_GetMemUsage(&zstdProps);
840 const UInt64 numZipThreads = options._memUsage_Compress / (mem1 + kMemPerThread);
841 if (numThreads > numZipThreads)
842 numThreads = (UInt32)numZipThreads;
843 }
844 return numThreads;
845 }
846 {
847 // threads for ZSTD are not fixed
848
849 // calculate estimated required number of ZST threads per file size statistics
850 UInt32 t = MY_ZSTDMT_NBWORKERS_MAX;
851 {
852 UInt64 averageNumberOfBlocks = 0;
853 const UInt64 averageSize = numBytesToCompress / numFilesToCompress;
854 const UInt64 jobSize = zstdProps.jobSize;
855 if (jobSize != 0)
856 averageNumberOfBlocks = averageSize / jobSize + 0;
857 if (t > averageNumberOfBlocks)
858 t = (UInt32)averageNumberOfBlocks;
859 }
860 if (t > numThreads)
861 t = numThreads;
862
863 // calculate the nuber of zip threads
864 UInt32 numZipThreads = numThreads;
865 if (t > 1)
866 numZipThreads = numThreads / t;
867 if (numZipThreads > numZipThreads_limit)
868 numZipThreads = numZipThreads_limit;
869 if (numZipThreads < 1)
870 numZipThreads = 1;
871 {
872 // recalculate the number of ZSTD threads via the number of ZIP threads
873 const UInt32 t2 = numThreads / numZipThreads;
874 if (t < t2)
875 t = t2;
876 }
877
878 if (options._memUsage_WasSet
879 && !options._numThreads_WasForced)
880 {
881 t = CalcThreads_for_ZipZstd(&zstdProps,
882 options._memUsage_Compress, numThreads, numZipThreads);
883 numThreads = numZipThreads;
884 }
885 // we don't use (nbWorkers = 1) here
886 if (t <= 1)
887 t = 0;
888 oneMethodMain->AddProp_NumThreads(t);
889 return numThreads;
890 }
891 }
892 */
893
894 #endif
895
896
897
898
899 static HRESULT Update2(
900 DECL_EXTERNAL_CODECS_LOC_VARS
901 COutArchive &archive,
902 CInArchive *inArchive,
903 const CObjectVector<CItemEx> &inputItems,
904 CObjectVector<CUpdateItem> &updateItems,
905 const CUpdateOptions &updateOptions,
906 const CCompressionMethodMode &options, bool outSeqMode,
907 const CByteBuffer *comment,
908 IArchiveUpdateCallback *updateCallback)
909 {
910 CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
911 updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
912
913 /*
914 CMyComPtr<IArchiveUpdateCallbackArcProp> reportArcProp;
915 updateCallback->QueryInterface(IID_IArchiveUpdateCallbackArcProp, (void **)&reportArcProp);
916 */
917
918 bool unknownComplexity = false;
919 UInt64 complexity = 0;
920 #ifndef Z7_ST
921 UInt64 numFilesToCompress = 0;
922 UInt64 numBytesToCompress = 0;
923 #endif
924
925 unsigned i;
926
927 for (i = 0; i < updateItems.Size(); i++)
928 {
929 const CUpdateItem &ui = updateItems[i];
930 if (ui.NewData)
931 {
932 if (ui.Size == (UInt64)(Int64)-1)
933 unknownComplexity = true;
934 else
935 complexity += ui.Size;
936 #ifndef Z7_ST
937 numBytesToCompress += ui.Size;
938 numFilesToCompress++;
939 #endif
940 /*
941 if (ui.Commented)
942 complexity += ui.CommentRange.Size;
943 */
944 }
945 else
946 {
947 CItemEx inputItem = inputItems[(unsigned)ui.IndexInArc];
948 if (inArchive->Read_LocalItem_After_CdItem_Full(inputItem) != S_OK)
949 return E_NOTIMPL;
950 complexity += inputItem.GetLocalFullSize();
951 // complexity += inputItem.GetCentralExtraPlusCommentSize();
952 }
953 complexity += kLocalHeaderSize;
954 complexity += kCentralHeaderSize;
955 }
956
957 if (comment)
958 complexity += comment->Size();
959 complexity++; // end of central
960
961 if (!unknownComplexity)
962 updateCallback->SetTotal(complexity);
963
964 UInt64 totalComplexity = complexity;
965
966 CCompressionMethodMode options2 = options;
967
968 if (options2._methods.IsEmpty())
969 {
970 // we need method item, if default method was used
971 options2._methods.AddNew();
972 }
973
974 CAddCommon compressor;
975 compressor.SetOptions(options2);
976
977 complexity = 0;
978
979 const Byte method = options.MethodSequence.FrontItem();
980
981 COneMethodInfo *oneMethodMain = NULL;
982 if (!options2._methods.IsEmpty())
983 oneMethodMain = &options2._methods[0];
984
985 {
986 FOR_VECTOR (mi, options2._methods)
987 {
988 options2.SetGlobalLevelTo(options2._methods[mi]);
989 }
990 }
991
992 if (oneMethodMain)
993 {
994 // appnote recommends to use EOS marker for LZMA.
995 if (method == NFileHeader::NCompressionMethod::kLZMA)
996 oneMethodMain->AddProp_EndMarker_if_NotFound(true);
997 }
998
999
1000 #ifndef Z7_ST
1001
1002 UInt32 numThreads = options._numThreads;
1003
1004 UInt32 numZipThreads_limit = numThreads;
1005 if (numZipThreads_limit > numFilesToCompress)
1006 numZipThreads_limit = (UInt32)numFilesToCompress;
1007
1008 if (numZipThreads_limit > 1)
1009 {
1010 const unsigned numFiles_OPEN_MAX = NSystem::Get_File_OPEN_MAX_Reduced_for_3_tasks();
1011 // printf("\nzip:numFiles_OPEN_MAX =%d\n", (unsigned)numFiles_OPEN_MAX);
1012 if (numZipThreads_limit > numFiles_OPEN_MAX)
1013 numZipThreads_limit = (UInt32)numFiles_OPEN_MAX;
1014 }
1015
1016 {
1017 const UInt32 kNumMaxThreads =
1018 #ifdef _WIN32
1019 64; // _WIN32 supports only 64 threads in one group. So no need for more threads here
1020 #else
1021 128;
1022 #endif
1023 if (numThreads > kNumMaxThreads)
1024 numThreads = kNumMaxThreads;
1025 }
1026 /*
1027 if (numThreads > MAXIMUM_WAIT_OBJECTS) // is 64 in Windows
1028 numThreads = MAXIMUM_WAIT_OBJECTS;
1029 */
1030
1031
1032 /*
1033 // zstd supports (numThreads == 0);
1034 if (numThreads < 1)
1035 numThreads = 1;
1036 */
1037
1038 bool mtMode = (numThreads > 1);
1039
1040 if (numFilesToCompress <= 1)
1041 mtMode = false;
1042
1043 // mtMode = true; // debug: to test mtMode
1044
1045 if (!mtMode)
1046 {
1047 // if (oneMethodMain) {
1048 /*
1049 if (method == NFileHeader::NCompressionMethod::kZstdWz)
1050 {
1051 if (oneMethodMain->FindProp(NCoderPropID::kNumThreads) < 0)
1052 {
1053 // numZstdThreads was not forced in oneMethodMain
1054 if (numThreads >= 1
1055 && options._memUsage_WasSet
1056 && !options._numThreads_WasForced)
1057 {
1058 NCompress::NZstd::CEncoderProps encoderProps;
1059 RINOK(encoderProps.SetFromMethodProps(*oneMethodMain))
1060 CZstdEncProps &zstdProps = encoderProps.EncProps;
1061 ZstdEncProps_NormalizeFull(&zstdProps);
1062 numThreads = ZstdEncProps_GetNumThreads_for_MemUsageLimit(
1063 &zstdProps, options._memUsage_Compress, numThreads);
1064 // we allow (nbWorkers = 1) here.
1065 }
1066 oneMethodMain->AddProp_NumThreads(numThreads);
1067 }
1068 } // kZstdWz
1069 */
1070 // } // oneMethodMain
1071
1072 FOR_VECTOR (mi, options2._methods)
1073 {
1074 COneMethodInfo &onem = options2._methods[mi];
1075
1076 if (onem.FindProp(NCoderPropID::kNumThreads) < 0)
1077 {
1078 // fixme: we should check the number of threads for xz method also
1079 // fixed for 9.31. bzip2 default is just one thread.
1080 onem.AddProp_NumThreads(numThreads);
1081 }
1082 }
1083 }
1084 else // mtMode
1085 {
1086 if (method == NFileHeader::NCompressionMethod::kStore && !options.Password_Defined)
1087 numThreads = 1;
1088
1089 if (oneMethodMain)
1090 {
1091
1092 if (method == NFileHeader::NCompressionMethod::kBZip2)
1093 {
1094 bool fixedNumber;
1095 UInt32 numBZip2Threads = oneMethodMain->Get_BZip2_NumThreads(fixedNumber);
1096 if (!fixedNumber)
1097 {
1098 const UInt64 averageSize = numBytesToCompress / numFilesToCompress;
1099 const UInt32 blockSize = oneMethodMain->Get_BZip2_BlockSize();
1100 const UInt64 averageNumberOfBlocks = averageSize / blockSize + 1;
1101 numBZip2Threads = 64;
1102 if (numBZip2Threads > averageNumberOfBlocks)
1103 numBZip2Threads = (UInt32)averageNumberOfBlocks;
1104 if (numBZip2Threads > numThreads)
1105 numBZip2Threads = numThreads;
1106 oneMethodMain->AddProp_NumThreads(numBZip2Threads);
1107 }
1108 numThreads /= numBZip2Threads;
1109 }
1110 else if (method == NFileHeader::NCompressionMethod::kXz)
1111 {
1112 UInt32 numLzmaThreads = 1;
1113 int numXzThreads = oneMethodMain->Get_Xz_NumThreads(numLzmaThreads);
1114 if (numXzThreads < 0)
1115 {
1116 // numXzThreads is unknown
1117 const UInt64 averageSize = numBytesToCompress / numFilesToCompress;
1118 const UInt64 blockSize = oneMethodMain->Get_Xz_BlockSize();
1119 UInt64 averageNumberOfBlocks = 1;
1120 if (blockSize != (UInt64)(Int64)-1)
1121 averageNumberOfBlocks = averageSize / blockSize + 1;
1122 UInt32 t = 256;
1123 if (t > averageNumberOfBlocks)
1124 t = (UInt32)averageNumberOfBlocks;
1125 t *= numLzmaThreads;
1126 if (t > numThreads)
1127 t = numThreads;
1128 oneMethodMain->AddProp_NumThreads(t);
1129 numXzThreads = (int)t;
1130 }
1131 numThreads /= (unsigned)numXzThreads;
1132 }
1133 /*
1134 else if (method == NFileHeader::NCompressionMethod::kZstdWz)
1135 {
1136 numThreads = SetZstdThreads(options,
1137 oneMethodMain, numThreads,
1138 numZipThreads_limit,
1139 numFilesToCompress, numBytesToCompress);
1140 }
1141 */
1142 else if (
1143 method == NFileHeader::NCompressionMethod::kDeflate
1144 || method == NFileHeader::NCompressionMethod::kDeflate64
1145 || method == NFileHeader::NCompressionMethod::kPPMd)
1146 {
1147 if (numThreads > 1
1148 && options._memUsage_WasSet
1149 && !options._numThreads_WasForced)
1150 {
1151 UInt64 methodMemUsage;
1152 if (method == NFileHeader::NCompressionMethod::kPPMd)
1153 methodMemUsage = oneMethodMain->Get_Ppmd_MemSize();
1154 else
1155 methodMemUsage = (4 << 20); // for deflate
1156 const UInt64 threadMemUsage = kMemPerThread + methodMemUsage;
1157 const UInt64 numThreads64 = options._memUsage_Compress / threadMemUsage;
1158 if (numThreads64 < numThreads)
1159 numThreads = (UInt32)numThreads64;
1160 }
1161 }
1162 else if (method == NFileHeader::NCompressionMethod::kLZMA)
1163 {
1164 // we suppose that default LZMA is 2 thread. So we don't change it
1165 const UInt32 numLZMAThreads = oneMethodMain->Get_Lzma_NumThreads();
1166 numThreads /= numLZMAThreads;
1167
1168 if (numThreads > 1
1169 && options._memUsage_WasSet
1170 && !options._numThreads_WasForced)
1171 {
1172 const UInt64 methodMemUsage = oneMethodMain->Get_Lzma_MemUsage(true);
1173 const UInt64 threadMemUsage = kMemPerThread + methodMemUsage;
1174 const UInt64 numThreads64 = options._memUsage_Compress / threadMemUsage;
1175 if (numThreads64 < numThreads)
1176 numThreads = (UInt32)numThreads64;
1177 }
1178 }
1179 } // (oneMethodMain)
1180
1181 if (numThreads > numZipThreads_limit)
1182 numThreads = numZipThreads_limit;
1183 if (numThreads <= 1)
1184 {
1185 mtMode = false;
1186 numThreads = 1;
1187 }
1188 }
1189
1190 // mtMode = true; // to test mtMode for seqMode
1191
1192 if (!mtMode)
1193 #endif
1194 return Update2St(
1195 EXTERNAL_CODECS_LOC_VARS
1196 archive, inArchive,
1197 inputItems, updateItems,
1198 updateOptions,
1199 &options2, outSeqMode,
1200 comment, updateCallback, totalComplexity,
1201 opCallback
1202 // , reportArcProp
1203 );
1204
1205
1206 #ifndef Z7_ST
1207
1208 /*
1209 CTotalStats stat;
1210 stat.Size = 0;
1211 stat.PackSize = 0;
1212 */
1213 if (numThreads < 1)
1214 numThreads = 1;
1215
1216 CObjectVector<CItemOut> items;
1217
1218 CMtProgressMixer *mtProgressMixerSpec = new CMtProgressMixer;
1219 CMyComPtr<ICompressProgressInfo> progress = mtProgressMixerSpec;
1220 mtProgressMixerSpec->Create(updateCallback, true);
1221
1222 CMtCompressProgressMixer mtCompressProgressMixer;
1223 mtCompressProgressMixer.Init(numThreads, mtProgressMixerSpec->RatioProgress);
1224
1225 CMemBlockManagerMt memManager(kBlockSize);
1226 CMemRefs refs(&memManager);
1227
1228 CMtSem mtSem;
1229 CThreads threads;
1230 mtSem.Head = -1;
1231 mtSem.Indexes.ClearAndSetSize(numThreads);
1232 {
1233 WRes wres = mtSem.Semaphore.Create(0, numThreads);
1234 if (wres != 0)
1235 return HRESULT_FROM_WIN32(wres);
1236 }
1237
1238 CUIntVector threadIndices; // list threads in order of updateItems
1239
1240 {
1241 RINOK(memManager.AllocateSpaceAlways((size_t)numThreads * (kMemPerThread / kBlockSize)))
1242 for (i = 0; i < updateItems.Size(); i++)
1243 refs.Refs.Add(CMemBlocks2());
1244
1245 for (i = 0; i < numThreads; i++)
1246 {
1247 threads.Threads.AddNew();
1248 // mtSem.Indexes[i] = -1; // actually we don't use these values
1249 }
1250
1251 for (i = 0; i < numThreads; i++)
1252 {
1253 CThreadInfo &threadInfo = threads.Threads[i];
1254 threadInfo.ThreadIndex = i;
1255 threadInfo.SetOptions(options2);
1256 #ifdef Z7_EXTERNAL_CODECS
1257 threadInfo._externalCodecs = _externalCodecs;
1258 #endif
1259 RINOK(threadInfo.CreateEvents())
1260 threadInfo.OutStreamSpec = new COutMemStream(&memManager);
1261 RINOK(threadInfo.OutStreamSpec->CreateEvents(SYNC_WFMO(&memManager.Synchro)))
1262 threadInfo.OutStream = threadInfo.OutStreamSpec;
1263 threadInfo.ProgressSpec = new CMtCompressProgress();
1264 threadInfo.Progress = threadInfo.ProgressSpec;
1265 threadInfo.ProgressSpec->Init(&mtCompressProgressMixer, i);
1266 threadInfo.MtSem = &mtSem;
1267 RINOK(threadInfo.CreateThread())
1268 }
1269 }
1270
1271 unsigned mtItemIndex = 0;
1272 unsigned itemIndex = 0;
1273 int lastRealStreamItemIndex = -1;
1274
1275
1276 while (itemIndex < updateItems.Size())
1277 {
1278 if (threadIndices.Size() < numThreads && mtItemIndex < updateItems.Size())
1279 {
1280 // we start ahead the threads for compressing
1281 // also we set refs.Refs[itemIndex].SeqMode that is used later
1282 // don't move that code block
1283
1284 CUpdateItem &ui = updateItems[mtItemIndex++];
1285 if (!ui.NewData)
1286 continue;
1287 CItemEx itemEx;
1288 CItemOut item;
1289
1290 if (ui.NewProps)
1291 {
1292 if (ui.IsDir)
1293 continue;
1294 }
1295 else
1296 {
1297 itemEx = inputItems[(unsigned)ui.IndexInArc];
1298 if (inArchive->Read_LocalItem_After_CdItem_Full(itemEx) != S_OK)
1299 return E_NOTIMPL;
1300 (CItem &)item = itemEx;
1301 if (item.IsDir() != ui.IsDir)
1302 return E_NOTIMPL;
1303 if (ui.IsDir)
1304 continue;
1305 }
1306
1307 CMyComPtr<ISequentialInStream> fileInStream;
1308
1309 CMemBlocks2 &memRef2 = refs.Refs[mtItemIndex - 1];
1310
1311 {
1312 NWindows::NSynchronization::CCriticalSectionLock lock(mtProgressMixerSpec->Mixer2->CriticalSection);
1313 const HRESULT res = updateCallback->GetStream(ui.IndexInClient, &fileInStream);
1314 if (res == S_FALSE)
1315 {
1316 complexity += ui.Size;
1317 complexity += kLocalHeaderSize;
1318 mtProgressMixerSpec->Mixer2->SetProgressOffset_NoLock(complexity);
1319 RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK))
1320 memRef2.Skip = true;
1321 continue;
1322 }
1323 RINOK(res)
1324 if (!fileInStream)
1325 return E_INVALIDARG;
1326 UpdatePropsFromStream(updateOptions, ui, fileInStream, updateCallback, totalComplexity);
1327 RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK))
1328 }
1329
1330 UInt32 k;
1331 for (k = 0; k < numThreads; k++)
1332 if (threads.Threads[k].IsFree)
1333 break;
1334
1335 if (k == numThreads)
1336 return E_FAIL;
1337 {
1338 {
1339 CThreadInfo &threadInfo = threads.Threads[k];
1340 threadInfo.IsFree = false;
1341 threadInfo.InStream = fileInStream;
1342
1343 bool inSeqMode = false;
1344
1345 if (!inSeqMode)
1346 {
1347 CMyComPtr<IInStream> inStream2;
1348 fileInStream->QueryInterface(IID_IInStream, (void **)&inStream2);
1349 inSeqMode = (inStream2 == NULL);
1350 }
1351 memRef2.InSeqMode = inSeqMode;
1352
1353 // !!!!! we must release ref before sending event
1354 // BUG was here in v4.43 and v4.44. It could change ref counter in two threads in same time
1355 fileInStream.Release();
1356
1357 threadInfo.OutStreamSpec->Init();
1358 threadInfo.ProgressSpec->Reinit();
1359
1360 threadInfo.UpdateIndex = mtItemIndex - 1;
1361 threadInfo.InSeqMode = inSeqMode;
1362 threadInfo.OutSeqMode = outSeqMode;
1363 threadInfo.FileTime = ui.Time; // FileTime is used for ZipCrypto only in seqMode
1364 threadInfo.ExpectedDataSize = ui.Size;
1365 threadInfo.ExpectedDataSize_IsConfirmed = ui.Size_WasSetFromStream;
1366
1367 threadInfo.CompressEvent.Set();
1368
1369 threadIndices.Add(k);
1370 }
1371 }
1372
1373 continue;
1374 }
1375
1376 if (refs.Refs[itemIndex].Skip)
1377 {
1378 itemIndex++;
1379 continue;
1380 }
1381
1382 const CUpdateItem &ui = updateItems[itemIndex];
1383
1384 CItemEx itemEx;
1385 CItemOut item;
1386
1387 if (!ui.NewProps || !ui.NewData)
1388 {
1389 itemEx = inputItems[(unsigned)ui.IndexInArc];
1390 if (inArchive->Read_LocalItem_After_CdItem_Full(itemEx) != S_OK)
1391 return E_NOTIMPL;
1392 (CItem &)item = itemEx;
1393 }
1394
1395 if (ui.NewData)
1396 {
1397 // bool isDir = ((ui.NewProps) ? ui.IsDir : item.IsDir());
1398 const bool isDir = ui.IsDir;
1399
1400 if (isDir)
1401 {
1402 RINOK(WriteDirHeader(archive, &options, ui, item))
1403 }
1404 else
1405 {
1406 CMemBlocks2 &memRef = refs.Refs[itemIndex];
1407
1408 if (memRef.Finished)
1409 {
1410 if (lastRealStreamItemIndex < (int)itemIndex)
1411 lastRealStreamItemIndex = (int)itemIndex;
1412
1413 SetFileHeader(options, ui, memRef.CompressingResult.DescriptorMode, item);
1414
1415 // the BUG was fixed in 9.26:
1416 // SetItemInfoFromCompressingResult must be after SetFileHeader
1417 // to write correct Size.
1418
1419 SetItemInfoFromCompressingResult(memRef.CompressingResult,
1420 options.IsRealAesMode(), options.AesKeyMode, item);
1421 RINOK(archive.ClearRestriction())
1422 archive.WriteLocalHeader(item);
1423 // RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK));
1424 CMyComPtr<ISequentialOutStream> outStream;
1425 archive.CreateStreamForCopying(outStream);
1426 memRef.WriteToStream(memManager.GetBlockSize(), outStream);
1427 // v23: we fixed the bug: we need to write descriptor also
1428 if (item.HasDescriptor())
1429 {
1430 /* that function doesn't rewrite local header, if item.HasDescriptor().
1431 it just writes descriptor */
1432 archive.WriteLocalHeader_Replace(item);
1433 }
1434 else
1435 archive.MoveCurPos(item.PackSize);
1436 memRef.FreeOpt(&memManager);
1437 /*
1438 if (reportArcProp)
1439 {
1440 stat.UpdateWithItem(item);
1441 RINOK(ReportProps(reportArcProp, ui.IndexInClient, item, options.IsRealAesMode()));
1442 }
1443 */
1444 }
1445 else
1446 {
1447 // current file was not finished
1448
1449 if (lastRealStreamItemIndex < (int)itemIndex)
1450 {
1451 // LocalHeader was not written for current itemIndex still
1452
1453 lastRealStreamItemIndex = (int)itemIndex;
1454
1455 // thread was started before for that item already, and memRef.SeqMode was set
1456
1457 CCompressingResult compressingResult;
1458 RINOK(compressor.Set_Pre_CompressionResult(
1459 memRef.InSeqMode, outSeqMode,
1460 ui.Size,
1461 compressingResult))
1462
1463 memRef.PreDescriptorMode = compressingResult.DescriptorMode;
1464 SetFileHeader(options, ui, compressingResult.DescriptorMode, item);
1465
1466 SetItemInfoFromCompressingResult(compressingResult, options.IsRealAesMode(), options.AesKeyMode, item);
1467
1468 // file Size can be 64-bit !!!
1469 RINOK(archive.SetRestrictionFromCurrent())
1470 archive.WriteLocalHeader(item);
1471 }
1472
1473 {
1474 CThreadInfo &thread = threads.Threads[threadIndices.FrontItem()];
1475 if (!thread.OutStreamSpec->WasUnlockEventSent())
1476 {
1477 CMyComPtr<IOutStream> outStream;
1478 archive.CreateStreamForCompressing(outStream);
1479 thread.OutStreamSpec->SetOutStream(outStream);
1480 thread.OutStreamSpec->SetRealStreamMode();
1481 }
1482 }
1483
1484 const WRes wres = mtSem.Semaphore.Lock();
1485 if (wres != 0)
1486 return HRESULT_FROM_WIN32(wres);
1487
1488 const int ti = mtSem.GetFreeItem();
1489 if (ti < 0)
1490 return E_FAIL;
1491
1492 CThreadInfo &threadInfo = threads.Threads[(unsigned)ti];
1493 threadInfo.InStream.Release();
1494 threadInfo.IsFree = true;
1495 RINOK(threadInfo.Result)
1496
1497 unsigned t = 0;
1498
1499 for (;;)
1500 {
1501 if (t == threadIndices.Size())
1502 return E_FAIL;
1503 if (threadIndices[t] == (unsigned)ti)
1504 break;
1505 t++;
1506 }
1507 threadIndices.Delete(t);
1508
1509 if (t == 0)
1510 {
1511 // if thread for current file was finished.
1512 if (threadInfo.UpdateIndex != itemIndex)
1513 return E_FAIL;
1514
1515 if (memRef.PreDescriptorMode != threadInfo.CompressingResult.DescriptorMode)
1516 return E_FAIL;
1517
1518 RINOK(threadInfo.OutStreamSpec->WriteToRealStream())
1519 threadInfo.OutStreamSpec->ReleaseOutStream();
1520 SetFileHeader(options, ui, threadInfo.CompressingResult.DescriptorMode, item);
1521 SetItemInfoFromCompressingResult(threadInfo.CompressingResult,
1522 options.IsRealAesMode(), options.AesKeyMode, item);
1523
1524 archive.WriteLocalHeader_Replace(item);
1525
1526 /*
1527 if (reportArcProp)
1528 {
1529 stat.UpdateWithItem(item);
1530 RINOK(ReportProps(reportArcProp, ui.IndexInClient, item, options.IsRealAesMode()));
1531 }
1532 */
1533 }
1534 else
1535 {
1536 // it's not current file. So we must store information in array
1537 CMemBlocks2 &memRef2 = refs.Refs[threadInfo.UpdateIndex];
1538 threadInfo.OutStreamSpec->DetachData(memRef2);
1539 memRef2.CompressingResult = threadInfo.CompressingResult;
1540 // memRef2.SeqMode = threadInfo.SeqMode; // it was set before
1541 memRef2.Finished = true;
1542 continue;
1543 }
1544 }
1545 }
1546 }
1547 else
1548 {
1549 RINOK(UpdateItemOldData(archive, inArchive, itemEx, ui, item, progress, opCallback, complexity))
1550 }
1551
1552 items.Add(item);
1553 complexity += kLocalHeaderSize;
1554 mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity);
1555 itemIndex++;
1556 }
1557
1558 RINOK(mtCompressProgressMixer.SetRatioInfo(0, NULL, NULL))
1559
1560 RINOK(archive.WriteCentralDir(items, comment))
1561
1562 /*
1563 if (reportArcProp)
1564 {
1565 RINOK(ReportArcProps(reportArcProp, stat));
1566 }
1567 */
1568
1569 complexity += kCentralHeaderSize * updateItems.Size() + 1;
1570 mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity);
1571 return mtCompressProgressMixer.SetRatioInfo(0, NULL, NULL);
1572
1573 #endif
1574 }
1575
1576 /*
1577 // we need CSeekOutStream, if we need Seek(0, STREAM_SEEK_CUR) for seqential stream
1578 Z7_CLASS_IMP_COM_1(
1579 CSeekOutStream
1580 , IOutStream
1581 )
1582 Z7_IFACE_COM7_IMP(ISequentialOutStream)
1583
1584 CMyComPtr<ISequentialOutStream> _seqStream;
1585 UInt64 _size;
1586 public:
1587 void Init(ISequentialOutStream *seqStream)
1588 {
1589 _size = 0;
1590 _seqStream = seqStream;
1591 }
1592 };
1593
1594 Z7_COM7F_IMF(CSeekOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize))
1595 {
1596 UInt32 realProcessedSize;
1597 const HRESULT result = _seqStream->Write(data, size, &realProcessedSize);
1598 _size += realProcessedSize;
1599 if (processedSize)
1600 *processedSize = realProcessedSize;
1601 return result;
1602 }
1603
1604 Z7_COM7F_IMF(CSeekOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition))
1605 {
1606 if (seekOrigin != STREAM_SEEK_CUR || offset != 0)
1607 return E_NOTIMPL;
1608 if (newPosition)
1609 *newPosition = (UInt64)_size;
1610 return S_OK;
1611 }
1612
1613 Z7_COM7F_IMF(CSeekOutStream::SetSize(UInt64 newSize))
1614 {
1615 UNUSED_VAR(newSize)
1616 return E_NOTIMPL;
1617 }
1618 */
1619
1620 static const size_t kCacheBlockSize = 1 << 20;
1621 static const size_t kCacheSize = kCacheBlockSize << 2;
1622 static const size_t kCacheMask = kCacheSize - 1;
1623
1624 Z7_CLASS_IMP_NOQIB_2(
1625 CCacheOutStream
1626 , IOutStream
1627 , IStreamSetRestriction
1628 )
1629 Z7_IFACE_COM7_IMP(ISequentialOutStream)
1630
1631 HRESULT _hres;
1632 CMyComPtr<ISequentialOutStream> _seqStream;
1633 CMyComPtr<IOutStream> _stream;
1634 CMyComPtr<IStreamSetRestriction> _setRestriction;
1635 Byte *_cache;
1636 size_t _cachedSize;
1637 UInt64 _cachedPos;
1638 UInt64 _virtPos;
1639 UInt64 _virtSize;
1640 UInt64 _phyPos;
1641 UInt64 _phySize;
1642 UInt64 _restrict_begin;
1643 UInt64 _restrict_end;
1644
1645 HRESULT FlushFromCache(size_t size);
1646 HRESULT FlushNonRestrictedBlocks();
1647 HRESULT FlushCache();
1648 HRESULT SetRestriction_ForWrite(size_t writeSize) const;
1649
1650 HRESULT SeekPhy(UInt64 pos)
1651 {
1652 if (pos == _phyPos)
1653 return S_OK;
1654 if (!_stream)
1655 return E_NOTIMPL;
1656 _hres = _stream->Seek((Int64)pos, STREAM_SEEK_SET, &_phyPos);
1657 if (_hres == S_OK && _phyPos != pos)
1658 _hres = E_FAIL;
1659 return _hres;
1660 }
1661
1662 public:
1663 CCacheOutStream(): _cache(NULL) {}
1664 ~CCacheOutStream();
1665 bool Allocate()
1666 {
1667 if (!_cache)
1668 _cache = (Byte *)::MidAlloc(kCacheSize);
1669 return _cache != NULL;
1670 }
1671 HRESULT Init(ISequentialOutStream *seqStream, IOutStream *stream, IStreamSetRestriction *setRestriction);
1672 HRESULT FinalFlush();
1673 };
1674
1675 CCacheOutStream::~CCacheOutStream()
1676 {
1677 ::MidFree(_cache);
1678 }
1679
1680
1681 HRESULT CCacheOutStream::Init(ISequentialOutStream *seqStream, IOutStream *stream, IStreamSetRestriction *setRestriction)
1682 {
1683 _hres = S_OK;
1684 _cachedSize = 0;
1685 _cachedPos = 0;
1686 _virtPos = 0;
1687 _virtSize = 0;
1688 // by default we have no restriction
1689 _restrict_begin = 0;
1690 _restrict_end = 0;
1691 _seqStream = seqStream;
1692 _stream = stream;
1693 _setRestriction = setRestriction;
1694 if (_stream)
1695 {
1696 RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &_virtPos))
1697 RINOK(_stream->Seek(0, STREAM_SEEK_END, &_virtSize))
1698 RINOK(_stream->Seek((Int64)_virtPos, STREAM_SEEK_SET, &_virtPos))
1699 }
1700 _phyPos = _virtPos;
1701 _phySize = _virtSize;
1702 return S_OK;
1703 }
1704
1705
1706 /* we call SetRestriction_ForWrite() just before Write() from cache.
1707 (_phyPos == _cachedPos)
1708 (writeSize != 0)
1709 */
1710 HRESULT CCacheOutStream::SetRestriction_ForWrite(size_t writeSize) const
1711 {
1712 if (!_setRestriction)
1713 return S_OK;
1714 PRF(printf("\n-- CCacheOutStream::SetRestriction_ForWrite _cachedPos = 0x%x, writeSize = %d\n", (unsigned)_cachedPos, (unsigned)writeSize));
1715 UInt64 begin = _restrict_begin;
1716 UInt64 end = _restrict_end;
1717 const UInt64 phyPos = _phyPos;
1718 if (phyPos != _cachedPos) return E_FAIL;
1719 if (phyPos == _phySize)
1720 {
1721 // The writing will be to the end of phy stream.
1722 // So we will try to use non-restricted write, if possible.
1723 if (begin == end)
1724 begin = _virtPos; // _virtSize; // it's supposed that (_virtSize == _virtPos)
1725 if (phyPos + writeSize <= begin)
1726 {
1727 // the write is not restricted
1728 PRF(printf("\n+++ write is not restricted \n"));
1729 begin = 0;
1730 end = 0;
1731 }
1732 else
1733 {
1734 if (begin > phyPos)
1735 begin = phyPos;
1736 end = (UInt64)(Int64)-1;
1737 }
1738 }
1739 else
1740 {
1741 // (phyPos != _phySize)
1742 if (begin == end || begin > phyPos)
1743 begin = phyPos;
1744 end = (UInt64)(Int64)-1;
1745 }
1746 return _setRestriction->SetRestriction(begin, end);
1747 }
1748
1749
1750 /* it writes up to (size) bytes from cache.
1751 (size > _cachedSize) is allowed
1752 */
1753 HRESULT CCacheOutStream::FlushFromCache(size_t size)
1754 {
1755 PRF(printf("\n-- CCacheOutStream::FlushFromCache %u\n", (unsigned)size));
1756 if (_hres != S_OK)
1757 return _hres;
1758 if (size > _cachedSize)
1759 size = _cachedSize;
1760 // (size <= _cachedSize)
1761 if (size == 0)
1762 return S_OK;
1763 RINOK(SeekPhy(_cachedPos))
1764 for (;;)
1765 {
1766 // (_phyPos == _cachedPos)
1767 const size_t pos = (size_t)_cachedPos & kCacheMask;
1768 const size_t cur = MyMin(kCacheSize - pos, size);
1769 _hres = SetRestriction_ForWrite(cur);
1770 RINOK(_hres)
1771 PRF(printf("\n-- CCacheOutStream::WriteFromCache _phyPos = 0x%x, size = %d\n", (unsigned)_phyPos, (unsigned)cur));
1772 _hres = WriteStream(_seqStream, _cache + pos, cur);
1773 RINOK(_hres)
1774 _phyPos += cur;
1775 if (_phySize < _phyPos)
1776 _phySize = _phyPos;
1777 _cachedPos += cur;
1778 _cachedSize -= cur;
1779 size -= cur;
1780 if (size == 0)
1781 return S_OK;
1782 }
1783 }
1784
1785
1786 HRESULT CCacheOutStream::FlushNonRestrictedBlocks()
1787 {
1788 for (;;)
1789 {
1790 const size_t size = kCacheBlockSize - ((size_t)_cachedPos & (kCacheBlockSize - 1));
1791 if (_cachedSize < size)
1792 break;
1793 UInt64 begin = _restrict_begin;
1794 if (begin == _restrict_end)
1795 begin = _virtPos;
1796 // we don't flush the data to restricted area
1797 if (_cachedPos + size > begin)
1798 break;
1799 RINOK(FlushFromCache(size))
1800 }
1801 return S_OK;
1802 }
1803
1804
1805 HRESULT CCacheOutStream::FlushCache()
1806 {
1807 return FlushFromCache(_cachedSize);
1808 }
1809
1810 HRESULT CCacheOutStream::FinalFlush()
1811 {
1812 _restrict_begin = 0;
1813 _restrict_end = 0;
1814 RINOK(FlushCache())
1815 if (_stream && _hres == S_OK)
1816 {
1817 if (_virtSize != _phySize)
1818 {
1819 // it's unexpected
1820 RINOK(_stream->SetSize(_virtSize))
1821 _phySize = _virtSize;
1822 }
1823 _hres = SeekPhy(_virtPos);
1824 }
1825 return _hres;
1826 }
1827
1828
1829 Z7_COM7F_IMF(CCacheOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize))
1830 {
1831 PRF(printf("\n==== CCacheOutStream::Write virtPos=0x%x, %u\n", (unsigned)_virtPos, (unsigned)size));
1832
1833 if (processedSize)
1834 *processedSize = 0;
1835 if (size == 0)
1836 return S_OK;
1837 if (_hres != S_OK)
1838 return _hres;
1839
1840 if (_cachedSize != 0)
1841 if (_virtPos < _cachedPos ||
1842 _virtPos > _cachedPos + _cachedSize)
1843 {
1844 RINOK(FlushCache())
1845 }
1846
1847 if (_cachedSize == 0)
1848 _cachedPos = _virtPos;
1849
1850 const size_t pos = (size_t)_virtPos & kCacheMask;
1851 {
1852 const size_t blockRem = kCacheBlockSize - ((size_t)_virtPos & (kCacheBlockSize - 1));
1853 if (size > blockRem)
1854 size = (UInt32)blockRem;
1855 }
1856 // _cachedPos <= _virtPos <= _cachedPos + _cachedSize
1857 const UInt64 cachedRem = _cachedPos + _cachedSize - _virtPos;
1858 if (cachedRem)
1859 {
1860 // _virtPos < _cachedPos + _cachedSize
1861 // we rewrite only existing data in cache. So _cachedSize will be not changed
1862 if (size > cachedRem)
1863 size = (UInt32)cachedRem;
1864 }
1865 else
1866 {
1867 // _virtPos == _cachedPos + _cachedSize
1868 // so we need to add new data to the end of cache
1869 if (_cachedSize == kCacheSize)
1870 {
1871 // cache is full. So we need to flush some part of cache.
1872 // we flush only one block, but we are allowed to flush any size here
1873 RINOK(FlushFromCache(kCacheBlockSize - ((size_t)_cachedPos & (kCacheBlockSize - 1))))
1874 }
1875 // _cachedSize != kCacheSize
1876 // so we have some space for new data in cache
1877 if (_cachedSize == 0)
1878 {
1879 /* this code is optional (for optimization):
1880 we write data directly without cache,
1881 if there is no restriction and we have full block. */
1882 if (_restrict_begin == _restrict_end
1883 && size == kCacheBlockSize)
1884 {
1885 RINOK(SeekPhy(_virtPos))
1886 if (_setRestriction)
1887 {
1888 _hres = _setRestriction->SetRestriction(_restrict_begin, _restrict_end);
1889 RINOK(_hres)
1890 }
1891 PRF(printf("\n-- CCacheOutStream::WriteDirectly _phyPos = 0x%x, size = %d\n", (unsigned)_phyPos, (unsigned)size));
1892 _hres = WriteStream(_seqStream, data, size);
1893 RINOK(_hres)
1894 if (processedSize)
1895 *processedSize = size;
1896 _virtPos += size;
1897 if (_virtSize < _virtPos)
1898 _virtSize = _virtPos;
1899 _phyPos += size;
1900 if (_phySize < _phyPos)
1901 _phySize = _phyPos;
1902 return S_OK;
1903 }
1904 }
1905 else // (_cachedSize != 0)
1906 {
1907 const size_t startPos = (size_t)_cachedPos & kCacheMask;
1908 // we don't allow new data to overwrite old start data in cache.
1909 // (startPos == pos) here means that cache is empty.
1910 // (startPos == pos) is not possible here.
1911 if (startPos > pos)
1912 size = (UInt32)MyMin((size_t)size, (size_t)(startPos - pos));
1913 }
1914 // _virtPos == (_cachedPos + _cachedSize) still
1915 _cachedSize += size;
1916 }
1917
1918 memcpy(_cache + pos, data, size);
1919 if (processedSize)
1920 *processedSize = size;
1921 _virtPos += size;
1922 if (_virtSize < _virtPos)
1923 _virtSize = _virtPos;
1924 return FlushNonRestrictedBlocks();
1925 }
1926
1927
1928 Z7_COM7F_IMF(CCacheOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition))
1929 {
1930 PRF(printf("\n==== CCacheOutStream::Seek seekOrigin=%d Seek =%u\n", seekOrigin, (unsigned)offset));
1931 switch (seekOrigin)
1932 {
1933 case STREAM_SEEK_SET: break;
1934 case STREAM_SEEK_CUR: offset += _virtPos; break;
1935 case STREAM_SEEK_END: offset += _virtSize; break;
1936 default: return STG_E_INVALIDFUNCTION;
1937 }
1938 if (offset < 0)
1939 return HRESULT_WIN32_ERROR_NEGATIVE_SEEK;
1940 _virtPos = (UInt64)offset;
1941 if (newPosition)
1942 *newPosition = (UInt64)offset;
1943 return S_OK;
1944 }
1945
1946
1947 Z7_COM7F_IMF(CCacheOutStream::SetSize(UInt64 newSize))
1948 {
1949 if (_hres != S_OK)
1950 return _hres;
1951
1952 if (newSize <= _cachedPos || _cachedSize == 0)
1953 {
1954 _cachedSize = 0;
1955 _cachedPos = newSize;
1956 }
1957 else
1958 {
1959 // _cachedSize != 0
1960 // newSize > _cachedPos
1961 const UInt64 offset = newSize - _cachedPos;
1962 if (offset <= _cachedSize)
1963 {
1964 // newSize is inside cached block or is touching cached block.
1965 // so we reduce cache
1966 _cachedSize = (size_t)offset;
1967 if (_phySize <= newSize)
1968 {
1969 // _phySize will be restored later after cache flush
1970 _virtSize = newSize;
1971 return S_OK;
1972 }
1973 // (_phySize > newSize)
1974 // so we must reduce phyStream size to (newSize) or to (_cachedPos)
1975 // newPhySize = _cachedPos; // optional reduce to _cachedPos
1976 }
1977 else
1978 {
1979 // newSize > _cachedPos + _cachedSize
1980 /* It's possible that we need to write zeros,
1981 if new size is larger than old size.
1982 We don't optimize for possible cases here.
1983 So we just flush the cache. */
1984 _hres = FlushCache();
1985 }
1986 }
1987
1988 _virtSize = newSize;
1989
1990 if (_hres != S_OK)
1991 return _hres;
1992
1993 if (newSize != _phySize)
1994 {
1995 if (!_stream)
1996 return E_NOTIMPL;
1997 // if (_phyPos > newSize)
1998 RINOK(SeekPhy(newSize))
1999 if (_setRestriction)
2000 {
2001 UInt64 begin = _restrict_begin;
2002 UInt64 end = _restrict_end;
2003 if (_cachedSize != 0)
2004 {
2005 if (begin > _cachedPos)
2006 begin = _cachedPos;
2007 end = (UInt64)(Int64)-1;
2008 }
2009 _hres = _setRestriction->SetRestriction(begin, end);
2010 RINOK(_hres)
2011 }
2012 _hres = _stream->SetSize(newSize);
2013 RINOK(_hres)
2014 _phySize = newSize;
2015 }
2016 return S_OK;
2017 }
2018
2019
2020 Z7_COM7F_IMF(CCacheOutStream::SetRestriction(UInt64 begin, UInt64 end))
2021 {
2022 PRF(printf("\n============ CCacheOutStream::SetRestriction 0x%x, %u\n", (unsigned)begin, (unsigned)end));
2023 _restrict_begin = begin;
2024 _restrict_end = end;
2025 return FlushNonRestrictedBlocks();
2026 }
2027
2028
2029
2030 HRESULT Update(
2031 DECL_EXTERNAL_CODECS_LOC_VARS
2032 const CObjectVector<CItemEx> &inputItems,
2033 CObjectVector<CUpdateItem> &updateItems,
2034 ISequentialOutStream *seqOutStream,
2035 CInArchive *inArchive, bool removeSfx,
2036 const CUpdateOptions &updateOptions,
2037 const CCompressionMethodMode &compressionMethodMode,
2038 IArchiveUpdateCallback *updateCallback)
2039 {
2040 /*
2041 // it was tested before
2042 if (inArchive)
2043 {
2044 if (!inArchive->CanUpdate())
2045 return E_NOTIMPL;
2046 }
2047 */
2048
2049 CMyComPtr<IStreamSetRestriction> setRestriction;
2050 seqOutStream->QueryInterface(IID_IStreamSetRestriction, (void **)&setRestriction);
2051 if (setRestriction)
2052 {
2053 RINOK(setRestriction->SetRestriction(0, 0))
2054 }
2055
2056 CMyComPtr<IOutStream> outStream;
2057 CCacheOutStream *cacheStream;
2058 bool outSeqMode;
2059
2060 {
2061 CMyComPtr<IOutStream> outStreamReal;
2062
2063 if (!compressionMethodMode.Force_SeqOutMode)
2064 {
2065 seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStreamReal);
2066 /*
2067 if (!outStreamReal)
2068 return E_NOTIMPL;
2069 */
2070 }
2071
2072 if (inArchive)
2073 {
2074 if (!inArchive->IsMultiVol && inArchive->ArcInfo.Base > 0 && !removeSfx)
2075 {
2076 IInStream *baseStream = inArchive->GetBaseStream();
2077 RINOK(InStream_SeekToBegin(baseStream))
2078 RINOK(NCompress::CopyStream_ExactSize(baseStream, seqOutStream, (UInt64)inArchive->ArcInfo.Base, NULL))
2079 }
2080 }
2081
2082 outSeqMode = (outStreamReal == NULL);
2083 if (outSeqMode)
2084 setRestriction.Release();
2085 /* CCacheOutStream works as non-restricted by default.
2086 So we use (setRestriction == NULL) for outSeqMode */
2087 // bool use_cacheStream = true;
2088 // if (use_cacheStream)
2089 {
2090 cacheStream = new CCacheOutStream();
2091 outStream = cacheStream;
2092 if (!cacheStream->Allocate())
2093 return E_OUTOFMEMORY;
2094 RINOK(cacheStream->Init(seqOutStream, outStreamReal, setRestriction))
2095 setRestriction.Release();
2096 if (!outSeqMode)
2097 setRestriction = cacheStream;
2098 }
2099 /*
2100 else if (!outStreamReal)
2101 {
2102 CSeekOutStream *seekOutStream = new CSeekOutStream();
2103 outStream = seekOutStream;
2104 seekOutStream->Init(seqOutStream);
2105 }
2106 else
2107 outStream = outStreamReal;
2108 */
2109 }
2110
2111 COutArchive outArchive;
2112 outArchive.SetRestriction = setRestriction;
2113
2114 RINOK(outArchive.Create(outStream))
2115
2116 if (inArchive)
2117 {
2118 if (!inArchive->IsMultiVol && (Int64)inArchive->ArcInfo.MarkerPos2 > inArchive->ArcInfo.Base)
2119 {
2120 IInStream *baseStream = inArchive->GetBaseStream();
2121 RINOK(InStream_SeekSet(baseStream, (UInt64)inArchive->ArcInfo.Base))
2122 const UInt64 embStubSize = (UInt64)((Int64)inArchive->ArcInfo.MarkerPos2 - inArchive->ArcInfo.Base);
2123 RINOK(NCompress::CopyStream_ExactSize(baseStream, outStream, embStubSize, NULL))
2124 outArchive.MoveCurPos(embStubSize);
2125 }
2126 }
2127
2128 RINOK (Update2(
2129 EXTERNAL_CODECS_LOC_VARS
2130 outArchive, inArchive,
2131 inputItems, updateItems,
2132 updateOptions,
2133 compressionMethodMode, outSeqMode,
2134 inArchive ? &inArchive->ArcInfo.Comment : NULL,
2135 updateCallback))
2136
2137 return cacheStream->FinalFlush();
2138 }
2139
2140 }}
2141