Merge with develop
This commit is contained in:
parent
79d11b8919
commit
6e471c02d3
8 changed files with 3 additions and 1490 deletions
|
@ -1,803 +0,0 @@
|
|||
/* LzFindMt.c -- multithreaded Match finder for LZ algorithms
|
||||
2015-10-15 : Igor Pavlov : Public domain */
|
||||
|
||||
#include "Precomp.h"
|
||||
|
||||
#include "LzHash.h"
|
||||
|
||||
#include "LzFindMt.h"
|
||||
|
||||
static void MtSync_Construct(CMtSync *p)
|
||||
{
|
||||
p->wasCreated = False;
|
||||
p->csWasInitialized = False;
|
||||
p->csWasEntered = False;
|
||||
Thread_Construct(&p->thread);
|
||||
Event_Construct(&p->canStart);
|
||||
Event_Construct(&p->wasStarted);
|
||||
Event_Construct(&p->wasStopped);
|
||||
Semaphore_Construct(&p->freeSemaphore);
|
||||
Semaphore_Construct(&p->filledSemaphore);
|
||||
}
|
||||
|
||||
static void MtSync_GetNextBlock(CMtSync *p)
|
||||
{
|
||||
if (p->needStart)
|
||||
{
|
||||
p->numProcessedBlocks = 1;
|
||||
p->needStart = False;
|
||||
p->stopWriting = False;
|
||||
p->exit = False;
|
||||
Event_Reset(&p->wasStarted);
|
||||
Event_Reset(&p->wasStopped);
|
||||
|
||||
Event_Set(&p->canStart);
|
||||
Event_Wait(&p->wasStarted);
|
||||
}
|
||||
else
|
||||
{
|
||||
CriticalSection_Leave(&p->cs);
|
||||
p->csWasEntered = False;
|
||||
p->numProcessedBlocks++;
|
||||
Semaphore_Release1(&p->freeSemaphore);
|
||||
}
|
||||
Semaphore_Wait(&p->filledSemaphore);
|
||||
CriticalSection_Enter(&p->cs);
|
||||
p->csWasEntered = True;
|
||||
}
|
||||
|
||||
/* MtSync_StopWriting must be called if Writing was started */
|
||||
|
||||
static void MtSync_StopWriting(CMtSync *p)
|
||||
{
|
||||
UInt32 myNumBlocks = p->numProcessedBlocks;
|
||||
if (!Thread_WasCreated(&p->thread) || p->needStart)
|
||||
return;
|
||||
p->stopWriting = True;
|
||||
if (p->csWasEntered)
|
||||
{
|
||||
CriticalSection_Leave(&p->cs);
|
||||
p->csWasEntered = False;
|
||||
}
|
||||
Semaphore_Release1(&p->freeSemaphore);
|
||||
|
||||
Event_Wait(&p->wasStopped);
|
||||
|
||||
while (myNumBlocks++ != p->numProcessedBlocks)
|
||||
{
|
||||
Semaphore_Wait(&p->filledSemaphore);
|
||||
Semaphore_Release1(&p->freeSemaphore);
|
||||
}
|
||||
p->needStart = True;
|
||||
}
|
||||
|
||||
static void MtSync_Destruct(CMtSync *p)
|
||||
{
|
||||
if (Thread_WasCreated(&p->thread))
|
||||
{
|
||||
MtSync_StopWriting(p);
|
||||
p->exit = True;
|
||||
if (p->needStart)
|
||||
Event_Set(&p->canStart);
|
||||
Thread_Wait(&p->thread);
|
||||
Thread_Close(&p->thread);
|
||||
}
|
||||
if (p->csWasInitialized)
|
||||
{
|
||||
CriticalSection_Delete(&p->cs);
|
||||
p->csWasInitialized = False;
|
||||
}
|
||||
|
||||
Event_Close(&p->canStart);
|
||||
Event_Close(&p->wasStarted);
|
||||
Event_Close(&p->wasStopped);
|
||||
Semaphore_Close(&p->freeSemaphore);
|
||||
Semaphore_Close(&p->filledSemaphore);
|
||||
|
||||
p->wasCreated = False;
|
||||
}
|
||||
|
||||
#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; }
|
||||
|
||||
static SRes MtSync_Create2(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj, UInt32 numBlocks)
|
||||
{
|
||||
if (p->wasCreated)
|
||||
return SZ_OK;
|
||||
|
||||
RINOK_THREAD(CriticalSection_Init(&p->cs));
|
||||
p->csWasInitialized = True;
|
||||
|
||||
RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canStart));
|
||||
RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->wasStarted));
|
||||
RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->wasStopped));
|
||||
|
||||
RINOK_THREAD(Semaphore_Create(&p->freeSemaphore, numBlocks, numBlocks));
|
||||
RINOK_THREAD(Semaphore_Create(&p->filledSemaphore, 0, numBlocks));
|
||||
|
||||
p->needStart = True;
|
||||
|
||||
RINOK_THREAD(Thread_Create(&p->thread, startAddress, obj));
|
||||
p->wasCreated = True;
|
||||
return SZ_OK;
|
||||
}
|
||||
|
||||
static SRes MtSync_Create(CMtSync *p, THREAD_FUNC_TYPE startAddress, void *obj, UInt32 numBlocks)
|
||||
{
|
||||
SRes res = MtSync_Create2(p, startAddress, obj, numBlocks);
|
||||
if (res != SZ_OK)
|
||||
MtSync_Destruct(p);
|
||||
return res;
|
||||
}
|
||||
|
||||
void MtSync_Init(CMtSync *p) { p->needStart = True; }
|
||||
|
||||
#define kMtMaxValForNormalize 0xFFFFFFFF
|
||||
|
||||
#define DEF_GetHeads2(name, v, action) \
|
||||
static void GetHeads ## name(const Byte *p, UInt32 pos, \
|
||||
UInt32 *hash, UInt32 hashMask, UInt32 *heads, UInt32 numHeads, const UInt32 *crc) \
|
||||
{ action; for (; numHeads != 0; numHeads--) { \
|
||||
const UInt32 value = (v); p++; *heads++ = pos - hash[value]; hash[value] = pos++; } }
|
||||
|
||||
#define DEF_GetHeads(name, v) DEF_GetHeads2(name, v, ;)
|
||||
|
||||
DEF_GetHeads2(2, (p[0] | ((UInt32)p[1] << 8)), UNUSED_VAR(hashMask); UNUSED_VAR(crc); )
|
||||
DEF_GetHeads(3, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8)) & hashMask)
|
||||
DEF_GetHeads(4, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8) ^ (crc[p[3]] << 5)) & hashMask)
|
||||
DEF_GetHeads(4b, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8) ^ ((UInt32)p[3] << 16)) & hashMask)
|
||||
/* DEF_GetHeads(5, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8) ^ (crc[p[3]] << 5) ^ (crc[p[4]] << 3)) & hashMask) */
|
||||
|
||||
static void HashThreadFunc(CMatchFinderMt *mt)
|
||||
{
|
||||
CMtSync *p = &mt->hashSync;
|
||||
for (;;)
|
||||
{
|
||||
UInt32 numProcessedBlocks = 0;
|
||||
Event_Wait(&p->canStart);
|
||||
Event_Set(&p->wasStarted);
|
||||
for (;;)
|
||||
{
|
||||
if (p->exit)
|
||||
return;
|
||||
if (p->stopWriting)
|
||||
{
|
||||
p->numProcessedBlocks = numProcessedBlocks;
|
||||
Event_Set(&p->wasStopped);
|
||||
break;
|
||||
}
|
||||
|
||||
{
|
||||
CMatchFinder *mf = mt->MatchFinder;
|
||||
if (MatchFinder_NeedMove(mf))
|
||||
{
|
||||
CriticalSection_Enter(&mt->btSync.cs);
|
||||
CriticalSection_Enter(&mt->hashSync.cs);
|
||||
{
|
||||
const Byte *beforePtr = Inline_MatchFinder_GetPointerToCurrentPos(mf);
|
||||
ptrdiff_t offset;
|
||||
MatchFinder_MoveBlock(mf);
|
||||
offset = beforePtr - Inline_MatchFinder_GetPointerToCurrentPos(mf);
|
||||
mt->pointerToCurPos -= offset;
|
||||
mt->buffer -= offset;
|
||||
}
|
||||
CriticalSection_Leave(&mt->btSync.cs);
|
||||
CriticalSection_Leave(&mt->hashSync.cs);
|
||||
continue;
|
||||
}
|
||||
|
||||
Semaphore_Wait(&p->freeSemaphore);
|
||||
|
||||
MatchFinder_ReadIfRequired(mf);
|
||||
if (mf->pos > (kMtMaxValForNormalize - kMtHashBlockSize))
|
||||
{
|
||||
UInt32 subValue = (mf->pos - mf->historySize - 1);
|
||||
MatchFinder_ReduceOffsets(mf, subValue);
|
||||
MatchFinder_Normalize3(subValue, mf->hash + mf->fixedHashSize, (size_t)mf->hashMask + 1);
|
||||
}
|
||||
{
|
||||
UInt32 *heads = mt->hashBuf + ((numProcessedBlocks++) & kMtHashNumBlocksMask) * kMtHashBlockSize;
|
||||
UInt32 num = mf->streamPos - mf->pos;
|
||||
heads[0] = 2;
|
||||
heads[1] = num;
|
||||
if (num >= mf->numHashBytes)
|
||||
{
|
||||
num = num - mf->numHashBytes + 1;
|
||||
if (num > kMtHashBlockSize - 2)
|
||||
num = kMtHashBlockSize - 2;
|
||||
mt->GetHeadsFunc(mf->buffer, mf->pos, mf->hash + mf->fixedHashSize, mf->hashMask, heads + 2, num, mf->crc);
|
||||
heads[0] += num;
|
||||
}
|
||||
mf->pos += num;
|
||||
mf->buffer += num;
|
||||
}
|
||||
}
|
||||
|
||||
Semaphore_Release1(&p->filledSemaphore);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void MatchFinderMt_GetNextBlock_Hash(CMatchFinderMt *p)
|
||||
{
|
||||
MtSync_GetNextBlock(&p->hashSync);
|
||||
p->hashBufPosLimit = p->hashBufPos = ((p->hashSync.numProcessedBlocks - 1) & kMtHashNumBlocksMask) * kMtHashBlockSize;
|
||||
p->hashBufPosLimit += p->hashBuf[p->hashBufPos++];
|
||||
p->hashNumAvail = p->hashBuf[p->hashBufPos++];
|
||||
}
|
||||
|
||||
#define kEmptyHashValue 0
|
||||
|
||||
/* #define MFMT_GM_INLINE */
|
||||
|
||||
#ifdef MFMT_GM_INLINE
|
||||
|
||||
#define NO_INLINE MY_FAST_CALL
|
||||
|
||||
static Int32 NO_INLINE GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son,
|
||||
UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue,
|
||||
UInt32 *_distances, UInt32 _maxLen, const UInt32 *hash, Int32 limit, UInt32 size, UInt32 *posRes)
|
||||
{
|
||||
do
|
||||
{
|
||||
UInt32 *distances = _distances + 1;
|
||||
UInt32 curMatch = pos - *hash++;
|
||||
|
||||
CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1;
|
||||
CLzRef *ptr1 = son + (_cyclicBufferPos << 1);
|
||||
UInt32 len0 = 0, len1 = 0;
|
||||
UInt32 cutValue = _cutValue;
|
||||
UInt32 maxLen = _maxLen;
|
||||
for (;;)
|
||||
{
|
||||
UInt32 delta = pos - curMatch;
|
||||
if (cutValue-- == 0 || delta >= _cyclicBufferSize)
|
||||
{
|
||||
*ptr0 = *ptr1 = kEmptyHashValue;
|
||||
break;
|
||||
}
|
||||
{
|
||||
CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1);
|
||||
const Byte *pb = cur - delta;
|
||||
UInt32 len = (len0 < len1 ? len0 : len1);
|
||||
if (pb[len] == cur[len])
|
||||
{
|
||||
if (++len != lenLimit && pb[len] == cur[len])
|
||||
while (++len != lenLimit)
|
||||
if (pb[len] != cur[len])
|
||||
break;
|
||||
if (maxLen < len)
|
||||
{
|
||||
*distances++ = maxLen = len;
|
||||
*distances++ = delta - 1;
|
||||
if (len == lenLimit)
|
||||
{
|
||||
*ptr1 = pair[0];
|
||||
*ptr0 = pair[1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (pb[len] < cur[len])
|
||||
{
|
||||
*ptr1 = curMatch;
|
||||
ptr1 = pair + 1;
|
||||
curMatch = *ptr1;
|
||||
len1 = len;
|
||||
}
|
||||
else
|
||||
{
|
||||
*ptr0 = curMatch;
|
||||
ptr0 = pair;
|
||||
curMatch = *ptr0;
|
||||
len0 = len;
|
||||
}
|
||||
}
|
||||
}
|
||||
pos++;
|
||||
_cyclicBufferPos++;
|
||||
cur++;
|
||||
{
|
||||
UInt32 num = (UInt32)(distances - _distances);
|
||||
*_distances = num - 1;
|
||||
_distances += num;
|
||||
limit -= num;
|
||||
}
|
||||
}
|
||||
while (limit > 0 && --size != 0);
|
||||
*posRes = pos;
|
||||
return limit;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances)
|
||||
{
|
||||
UInt32 numProcessed = 0;
|
||||
UInt32 curPos = 2;
|
||||
UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2);
|
||||
|
||||
distances[1] = p->hashNumAvail;
|
||||
|
||||
while (curPos < limit)
|
||||
{
|
||||
if (p->hashBufPos == p->hashBufPosLimit)
|
||||
{
|
||||
MatchFinderMt_GetNextBlock_Hash(p);
|
||||
distances[1] = numProcessed + p->hashNumAvail;
|
||||
if (p->hashNumAvail >= p->numHashBytes)
|
||||
continue;
|
||||
distances[0] = curPos + p->hashNumAvail;
|
||||
distances += curPos;
|
||||
for (; p->hashNumAvail != 0; p->hashNumAvail--)
|
||||
*distances++ = 0;
|
||||
return;
|
||||
}
|
||||
{
|
||||
UInt32 size = p->hashBufPosLimit - p->hashBufPos;
|
||||
UInt32 lenLimit = p->matchMaxLen;
|
||||
UInt32 pos = p->pos;
|
||||
UInt32 cyclicBufferPos = p->cyclicBufferPos;
|
||||
if (lenLimit >= p->hashNumAvail)
|
||||
lenLimit = p->hashNumAvail;
|
||||
{
|
||||
UInt32 size2 = p->hashNumAvail - lenLimit + 1;
|
||||
if (size2 < size)
|
||||
size = size2;
|
||||
size2 = p->cyclicBufferSize - cyclicBufferPos;
|
||||
if (size2 < size)
|
||||
size = size2;
|
||||
}
|
||||
|
||||
#ifndef MFMT_GM_INLINE
|
||||
while (curPos < limit && size-- != 0)
|
||||
{
|
||||
UInt32 *startDistances = distances + curPos;
|
||||
UInt32 num = (UInt32)(GetMatchesSpec1(lenLimit, pos - p->hashBuf[p->hashBufPos++],
|
||||
pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue,
|
||||
startDistances + 1, p->numHashBytes - 1) - startDistances);
|
||||
*startDistances = num - 1;
|
||||
curPos += num;
|
||||
cyclicBufferPos++;
|
||||
pos++;
|
||||
p->buffer++;
|
||||
}
|
||||
#else
|
||||
{
|
||||
UInt32 posRes;
|
||||
curPos = limit - GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue,
|
||||
distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, (Int32)(limit - curPos), size, &posRes);
|
||||
p->hashBufPos += posRes - pos;
|
||||
cyclicBufferPos += posRes - pos;
|
||||
p->buffer += posRes - pos;
|
||||
pos = posRes;
|
||||
}
|
||||
#endif
|
||||
|
||||
numProcessed += pos - p->pos;
|
||||
p->hashNumAvail -= pos - p->pos;
|
||||
p->pos = pos;
|
||||
if (cyclicBufferPos == p->cyclicBufferSize)
|
||||
cyclicBufferPos = 0;
|
||||
p->cyclicBufferPos = cyclicBufferPos;
|
||||
}
|
||||
}
|
||||
|
||||
distances[0] = curPos;
|
||||
}
|
||||
|
||||
static void BtFillBlock(CMatchFinderMt *p, UInt32 globalBlockIndex)
|
||||
{
|
||||
CMtSync *sync = &p->hashSync;
|
||||
if (!sync->needStart)
|
||||
{
|
||||
CriticalSection_Enter(&sync->cs);
|
||||
sync->csWasEntered = True;
|
||||
}
|
||||
|
||||
BtGetMatches(p, p->btBuf + (globalBlockIndex & kMtBtNumBlocksMask) * kMtBtBlockSize);
|
||||
|
||||
if (p->pos > kMtMaxValForNormalize - kMtBtBlockSize)
|
||||
{
|
||||
UInt32 subValue = p->pos - p->cyclicBufferSize;
|
||||
MatchFinder_Normalize3(subValue, p->son, (size_t)p->cyclicBufferSize * 2);
|
||||
p->pos -= subValue;
|
||||
}
|
||||
|
||||
if (!sync->needStart)
|
||||
{
|
||||
CriticalSection_Leave(&sync->cs);
|
||||
sync->csWasEntered = False;
|
||||
}
|
||||
}
|
||||
|
||||
void BtThreadFunc(CMatchFinderMt *mt)
|
||||
{
|
||||
CMtSync *p = &mt->btSync;
|
||||
for (;;)
|
||||
{
|
||||
UInt32 blockIndex = 0;
|
||||
Event_Wait(&p->canStart);
|
||||
Event_Set(&p->wasStarted);
|
||||
for (;;)
|
||||
{
|
||||
if (p->exit)
|
||||
return;
|
||||
if (p->stopWriting)
|
||||
{
|
||||
p->numProcessedBlocks = blockIndex;
|
||||
MtSync_StopWriting(&mt->hashSync);
|
||||
Event_Set(&p->wasStopped);
|
||||
break;
|
||||
}
|
||||
Semaphore_Wait(&p->freeSemaphore);
|
||||
BtFillBlock(mt, blockIndex++);
|
||||
Semaphore_Release1(&p->filledSemaphore);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void MatchFinderMt_Construct(CMatchFinderMt *p)
|
||||
{
|
||||
p->hashBuf = NULL;
|
||||
MtSync_Construct(&p->hashSync);
|
||||
MtSync_Construct(&p->btSync);
|
||||
}
|
||||
|
||||
static void MatchFinderMt_FreeMem(CMatchFinderMt *p, ISzAlloc *alloc)
|
||||
{
|
||||
alloc->Free(alloc, p->hashBuf);
|
||||
p->hashBuf = NULL;
|
||||
}
|
||||
|
||||
void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAlloc *alloc)
|
||||
{
|
||||
MtSync_Destruct(&p->hashSync);
|
||||
MtSync_Destruct(&p->btSync);
|
||||
MatchFinderMt_FreeMem(p, alloc);
|
||||
}
|
||||
|
||||
#define kHashBufferSize (kMtHashBlockSize * kMtHashNumBlocks)
|
||||
#define kBtBufferSize (kMtBtBlockSize * kMtBtNumBlocks)
|
||||
|
||||
static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE HashThreadFunc2(void *p) { HashThreadFunc((CMatchFinderMt *)p); return 0; }
|
||||
static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE BtThreadFunc2(void *p)
|
||||
{
|
||||
Byte allocaDummy[0x180];
|
||||
unsigned i = 0;
|
||||
for (i = 0; i < 16; i++)
|
||||
allocaDummy[i] = (Byte)0;
|
||||
if (allocaDummy[0] == 0)
|
||||
BtThreadFunc((CMatchFinderMt *)p);
|
||||
return 0;
|
||||
}
|
||||
|
||||
SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddBufferBefore,
|
||||
UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAlloc *alloc)
|
||||
{
|
||||
CMatchFinder *mf = p->MatchFinder;
|
||||
p->historySize = historySize;
|
||||
if (kMtBtBlockSize <= matchMaxLen * 4)
|
||||
return SZ_ERROR_PARAM;
|
||||
if (!p->hashBuf)
|
||||
{
|
||||
p->hashBuf = (UInt32 *)alloc->Alloc(alloc, (kHashBufferSize + kBtBufferSize) * sizeof(UInt32));
|
||||
if (!p->hashBuf)
|
||||
return SZ_ERROR_MEM;
|
||||
p->btBuf = p->hashBuf + kHashBufferSize;
|
||||
}
|
||||
keepAddBufferBefore += (kHashBufferSize + kBtBufferSize);
|
||||
keepAddBufferAfter += kMtHashBlockSize;
|
||||
if (!MatchFinder_Create(mf, historySize, keepAddBufferBefore, matchMaxLen, keepAddBufferAfter, alloc))
|
||||
return SZ_ERROR_MEM;
|
||||
|
||||
RINOK(MtSync_Create(&p->hashSync, HashThreadFunc2, p, kMtHashNumBlocks));
|
||||
RINOK(MtSync_Create(&p->btSync, BtThreadFunc2, p, kMtBtNumBlocks));
|
||||
return SZ_OK;
|
||||
}
|
||||
|
||||
/* Call it after ReleaseStream / SetStream */
|
||||
void MatchFinderMt_Init(CMatchFinderMt *p)
|
||||
{
|
||||
CMatchFinder *mf = p->MatchFinder;
|
||||
p->btBufPos = p->btBufPosLimit = 0;
|
||||
p->hashBufPos = p->hashBufPosLimit = 0;
|
||||
|
||||
/* Init without data reading. We don't want to read data in this thread */
|
||||
MatchFinder_Init_2(mf, False);
|
||||
|
||||
p->pointerToCurPos = Inline_MatchFinder_GetPointerToCurrentPos(mf);
|
||||
p->btNumAvailBytes = 0;
|
||||
p->lzPos = p->historySize + 1;
|
||||
|
||||
p->hash = mf->hash;
|
||||
p->fixedHashSize = mf->fixedHashSize;
|
||||
p->crc = mf->crc;
|
||||
|
||||
p->son = mf->son;
|
||||
p->matchMaxLen = mf->matchMaxLen;
|
||||
p->numHashBytes = mf->numHashBytes;
|
||||
p->pos = mf->pos;
|
||||
p->buffer = mf->buffer;
|
||||
p->cyclicBufferPos = mf->cyclicBufferPos;
|
||||
p->cyclicBufferSize = mf->cyclicBufferSize;
|
||||
p->cutValue = mf->cutValue;
|
||||
}
|
||||
|
||||
/* ReleaseStream is required to finish multithreading */
|
||||
void MatchFinderMt_ReleaseStream(CMatchFinderMt *p)
|
||||
{
|
||||
MtSync_StopWriting(&p->btSync);
|
||||
/* p->MatchFinder->ReleaseStream(); */
|
||||
}
|
||||
|
||||
static void MatchFinderMt_Normalize(CMatchFinderMt *p)
|
||||
{
|
||||
MatchFinder_Normalize3(p->lzPos - p->historySize - 1, p->hash, p->fixedHashSize);
|
||||
p->lzPos = p->historySize + 1;
|
||||
}
|
||||
|
||||
static void MatchFinderMt_GetNextBlock_Bt(CMatchFinderMt *p)
|
||||
{
|
||||
UInt32 blockIndex;
|
||||
MtSync_GetNextBlock(&p->btSync);
|
||||
blockIndex = ((p->btSync.numProcessedBlocks - 1) & kMtBtNumBlocksMask);
|
||||
p->btBufPosLimit = p->btBufPos = blockIndex * kMtBtBlockSize;
|
||||
p->btBufPosLimit += p->btBuf[p->btBufPos++];
|
||||
p->btNumAvailBytes = p->btBuf[p->btBufPos++];
|
||||
if (p->lzPos >= kMtMaxValForNormalize - kMtBtBlockSize)
|
||||
MatchFinderMt_Normalize(p);
|
||||
}
|
||||
|
||||
static const Byte * MatchFinderMt_GetPointerToCurrentPos(CMatchFinderMt *p)
|
||||
{
|
||||
return p->pointerToCurPos;
|
||||
}
|
||||
|
||||
#define GET_NEXT_BLOCK_IF_REQUIRED if (p->btBufPos == p->btBufPosLimit) MatchFinderMt_GetNextBlock_Bt(p);
|
||||
|
||||
static UInt32 MatchFinderMt_GetNumAvailableBytes(CMatchFinderMt *p)
|
||||
{
|
||||
GET_NEXT_BLOCK_IF_REQUIRED;
|
||||
return p->btNumAvailBytes;
|
||||
}
|
||||
|
||||
static UInt32 * MixMatches2(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances)
|
||||
{
|
||||
UInt32 h2, curMatch2;
|
||||
UInt32 *hash = p->hash;
|
||||
const Byte *cur = p->pointerToCurPos;
|
||||
UInt32 lzPos = p->lzPos;
|
||||
MT_HASH2_CALC
|
||||
|
||||
curMatch2 = hash[h2];
|
||||
hash[h2] = lzPos;
|
||||
|
||||
if (curMatch2 >= matchMinPos)
|
||||
if (cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0])
|
||||
{
|
||||
*distances++ = 2;
|
||||
*distances++ = lzPos - curMatch2 - 1;
|
||||
}
|
||||
|
||||
return distances;
|
||||
}
|
||||
|
||||
static UInt32 * MixMatches3(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances)
|
||||
{
|
||||
UInt32 h2, h3, curMatch2, curMatch3;
|
||||
UInt32 *hash = p->hash;
|
||||
const Byte *cur = p->pointerToCurPos;
|
||||
UInt32 lzPos = p->lzPos;
|
||||
MT_HASH3_CALC
|
||||
|
||||
curMatch2 = hash[ h2];
|
||||
curMatch3 = hash[kFix3HashSize + h3];
|
||||
|
||||
hash[ h2] = lzPos;
|
||||
hash[kFix3HashSize + h3] = lzPos;
|
||||
|
||||
if (curMatch2 >= matchMinPos && cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0])
|
||||
{
|
||||
distances[1] = lzPos - curMatch2 - 1;
|
||||
if (cur[(ptrdiff_t)curMatch2 - lzPos + 2] == cur[2])
|
||||
{
|
||||
distances[0] = 3;
|
||||
return distances + 2;
|
||||
}
|
||||
distances[0] = 2;
|
||||
distances += 2;
|
||||
}
|
||||
|
||||
if (curMatch3 >= matchMinPos && cur[(ptrdiff_t)curMatch3 - lzPos] == cur[0])
|
||||
{
|
||||
*distances++ = 3;
|
||||
*distances++ = lzPos - curMatch3 - 1;
|
||||
}
|
||||
|
||||
return distances;
|
||||
}
|
||||
|
||||
/*
|
||||
static UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances)
|
||||
{
|
||||
UInt32 h2, h3, h4, curMatch2, curMatch3, curMatch4;
|
||||
UInt32 *hash = p->hash;
|
||||
const Byte *cur = p->pointerToCurPos;
|
||||
UInt32 lzPos = p->lzPos;
|
||||
MT_HASH4_CALC
|
||||
|
||||
curMatch2 = hash[ h2];
|
||||
curMatch3 = hash[kFix3HashSize + h3];
|
||||
curMatch4 = hash[kFix4HashSize + h4];
|
||||
|
||||
hash[ h2] = lzPos;
|
||||
hash[kFix3HashSize + h3] = lzPos;
|
||||
hash[kFix4HashSize + h4] = lzPos;
|
||||
|
||||
if (curMatch2 >= matchMinPos && cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0])
|
||||
{
|
||||
distances[1] = lzPos - curMatch2 - 1;
|
||||
if (cur[(ptrdiff_t)curMatch2 - lzPos + 2] == cur[2])
|
||||
{
|
||||
distances[0] = (cur[(ptrdiff_t)curMatch2 - lzPos + 3] == cur[3]) ? 4 : 3;
|
||||
return distances + 2;
|
||||
}
|
||||
distances[0] = 2;
|
||||
distances += 2;
|
||||
}
|
||||
|
||||
if (curMatch3 >= matchMinPos && cur[(ptrdiff_t)curMatch3 - lzPos] == cur[0])
|
||||
{
|
||||
distances[1] = lzPos - curMatch3 - 1;
|
||||
if (cur[(ptrdiff_t)curMatch3 - lzPos + 3] == cur[3])
|
||||
{
|
||||
distances[0] = 4;
|
||||
return distances + 2;
|
||||
}
|
||||
distances[0] = 3;
|
||||
distances += 2;
|
||||
}
|
||||
|
||||
if (curMatch4 >= matchMinPos)
|
||||
if (
|
||||
cur[(ptrdiff_t)curMatch4 - lzPos] == cur[0] &&
|
||||
cur[(ptrdiff_t)curMatch4 - lzPos + 3] == cur[3]
|
||||
)
|
||||
{
|
||||
*distances++ = 4;
|
||||
*distances++ = lzPos - curMatch4 - 1;
|
||||
}
|
||||
|
||||
return distances;
|
||||
}
|
||||
*/
|
||||
|
||||
#define INCREASE_LZ_POS p->lzPos++; p->pointerToCurPos++;
|
||||
|
||||
static UInt32 MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *distances)
|
||||
{
|
||||
const UInt32 *btBuf = p->btBuf + p->btBufPos;
|
||||
UInt32 len = *btBuf++;
|
||||
p->btBufPos += 1 + len;
|
||||
p->btNumAvailBytes--;
|
||||
{
|
||||
UInt32 i;
|
||||
for (i = 0; i < len; i += 2)
|
||||
{
|
||||
*distances++ = *btBuf++;
|
||||
*distances++ = *btBuf++;
|
||||
}
|
||||
}
|
||||
INCREASE_LZ_POS
|
||||
return len;
|
||||
}
|
||||
|
||||
static UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *distances)
|
||||
{
|
||||
const UInt32 *btBuf = p->btBuf + p->btBufPos;
|
||||
UInt32 len = *btBuf++;
|
||||
p->btBufPos += 1 + len;
|
||||
|
||||
if (len == 0)
|
||||
{
|
||||
/* change for bt5 ! */
|
||||
if (p->btNumAvailBytes-- >= 4)
|
||||
len = (UInt32)(p->MixMatchesFunc(p, p->lzPos - p->historySize, distances) - (distances));
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Condition: there are matches in btBuf with length < p->numHashBytes */
|
||||
UInt32 *distances2;
|
||||
p->btNumAvailBytes--;
|
||||
distances2 = p->MixMatchesFunc(p, p->lzPos - btBuf[1], distances);
|
||||
do
|
||||
{
|
||||
*distances2++ = *btBuf++;
|
||||
*distances2++ = *btBuf++;
|
||||
}
|
||||
while ((len -= 2) != 0);
|
||||
len = (UInt32)(distances2 - (distances));
|
||||
}
|
||||
INCREASE_LZ_POS
|
||||
return len;
|
||||
}
|
||||
|
||||
#define SKIP_HEADER2_MT do { GET_NEXT_BLOCK_IF_REQUIRED
|
||||
#define SKIP_HEADER_MT(n) SKIP_HEADER2_MT if (p->btNumAvailBytes-- >= (n)) { const Byte *cur = p->pointerToCurPos; UInt32 *hash = p->hash;
|
||||
#define SKIP_FOOTER_MT } INCREASE_LZ_POS p->btBufPos += p->btBuf[p->btBufPos] + 1; } while (--num != 0);
|
||||
|
||||
static void MatchFinderMt0_Skip(CMatchFinderMt *p, UInt32 num)
|
||||
{
|
||||
SKIP_HEADER2_MT { p->btNumAvailBytes--;
|
||||
SKIP_FOOTER_MT
|
||||
}
|
||||
|
||||
static void MatchFinderMt2_Skip(CMatchFinderMt *p, UInt32 num)
|
||||
{
|
||||
SKIP_HEADER_MT(2)
|
||||
UInt32 h2;
|
||||
MT_HASH2_CALC
|
||||
hash[h2] = p->lzPos;
|
||||
SKIP_FOOTER_MT
|
||||
}
|
||||
|
||||
static void MatchFinderMt3_Skip(CMatchFinderMt *p, UInt32 num)
|
||||
{
|
||||
SKIP_HEADER_MT(3)
|
||||
UInt32 h2, h3;
|
||||
MT_HASH3_CALC
|
||||
hash[kFix3HashSize + h3] =
|
||||
hash[ h2] =
|
||||
p->lzPos;
|
||||
SKIP_FOOTER_MT
|
||||
}
|
||||
|
||||
/*
|
||||
static void MatchFinderMt4_Skip(CMatchFinderMt *p, UInt32 num)
|
||||
{
|
||||
SKIP_HEADER_MT(4)
|
||||
UInt32 h2, h3, h4;
|
||||
MT_HASH4_CALC
|
||||
hash[kFix4HashSize + h4] =
|
||||
hash[kFix3HashSize + h3] =
|
||||
hash[ h2] =
|
||||
p->lzPos;
|
||||
SKIP_FOOTER_MT
|
||||
}
|
||||
*/
|
||||
|
||||
void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable)
|
||||
{
|
||||
vTable->Init = (Mf_Init_Func)MatchFinderMt_Init;
|
||||
vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinderMt_GetNumAvailableBytes;
|
||||
vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinderMt_GetPointerToCurrentPos;
|
||||
vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt_GetMatches;
|
||||
|
||||
switch (p->MatchFinder->numHashBytes)
|
||||
{
|
||||
case 2:
|
||||
p->GetHeadsFunc = GetHeads2;
|
||||
p->MixMatchesFunc = (Mf_Mix_Matches)0;
|
||||
vTable->Skip = (Mf_Skip_Func)MatchFinderMt0_Skip;
|
||||
vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches;
|
||||
break;
|
||||
case 3:
|
||||
p->GetHeadsFunc = GetHeads3;
|
||||
p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches2;
|
||||
vTable->Skip = (Mf_Skip_Func)MatchFinderMt2_Skip;
|
||||
break;
|
||||
default:
|
||||
/* case 4: */
|
||||
p->GetHeadsFunc = p->MatchFinder->bigHash ? GetHeads4b : GetHeads4;
|
||||
p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches3;
|
||||
vTable->Skip = (Mf_Skip_Func)MatchFinderMt3_Skip;
|
||||
break;
|
||||
/*
|
||||
default:
|
||||
p->GetHeadsFunc = GetHeads5;
|
||||
p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches4;
|
||||
vTable->Skip = (Mf_Skip_Func)MatchFinderMt4_Skip;
|
||||
break;
|
||||
*/
|
||||
}
|
||||
}
|
|
@ -1,101 +0,0 @@
|
|||
/* LzFindMt.h -- multithreaded Match finder for LZ algorithms
|
||||
2015-05-03 : Igor Pavlov : Public domain */
|
||||
|
||||
#ifndef __LZ_FIND_MT_H
|
||||
#define __LZ_FIND_MT_H
|
||||
|
||||
#include "LzFind.h"
|
||||
#include "Threads.h"
|
||||
|
||||
EXTERN_C_BEGIN
|
||||
|
||||
#define kMtHashBlockSize (1 << 13)
|
||||
#define kMtHashNumBlocks (1 << 3)
|
||||
#define kMtHashNumBlocksMask (kMtHashNumBlocks - 1)
|
||||
|
||||
#define kMtBtBlockSize (1 << 14)
|
||||
#define kMtBtNumBlocks (1 << 6)
|
||||
#define kMtBtNumBlocksMask (kMtBtNumBlocks - 1)
|
||||
|
||||
typedef struct _CMtSync
|
||||
{
|
||||
Bool wasCreated;
|
||||
Bool needStart;
|
||||
Bool exit;
|
||||
Bool stopWriting;
|
||||
|
||||
CThread thread;
|
||||
CAutoResetEvent canStart;
|
||||
CAutoResetEvent wasStarted;
|
||||
CAutoResetEvent wasStopped;
|
||||
CSemaphore freeSemaphore;
|
||||
CSemaphore filledSemaphore;
|
||||
Bool csWasInitialized;
|
||||
Bool csWasEntered;
|
||||
CCriticalSection cs;
|
||||
UInt32 numProcessedBlocks;
|
||||
} CMtSync;
|
||||
|
||||
typedef UInt32 * (*Mf_Mix_Matches)(void *p, UInt32 matchMinPos, UInt32 *distances);
|
||||
|
||||
/* kMtCacheLineDummy must be >= size_of_CPU_cache_line */
|
||||
#define kMtCacheLineDummy 128
|
||||
|
||||
typedef void (*Mf_GetHeads)(const Byte *buffer, UInt32 pos,
|
||||
UInt32 *hash, UInt32 hashMask, UInt32 *heads, UInt32 numHeads, const UInt32 *crc);
|
||||
|
||||
typedef struct _CMatchFinderMt
|
||||
{
|
||||
/* LZ */
|
||||
const Byte *pointerToCurPos;
|
||||
UInt32 *btBuf;
|
||||
UInt32 btBufPos;
|
||||
UInt32 btBufPosLimit;
|
||||
UInt32 lzPos;
|
||||
UInt32 btNumAvailBytes;
|
||||
|
||||
UInt32 *hash;
|
||||
UInt32 fixedHashSize;
|
||||
UInt32 historySize;
|
||||
const UInt32 *crc;
|
||||
|
||||
Mf_Mix_Matches MixMatchesFunc;
|
||||
|
||||
/* LZ + BT */
|
||||
CMtSync btSync;
|
||||
Byte btDummy[kMtCacheLineDummy];
|
||||
|
||||
/* BT */
|
||||
UInt32 *hashBuf;
|
||||
UInt32 hashBufPos;
|
||||
UInt32 hashBufPosLimit;
|
||||
UInt32 hashNumAvail;
|
||||
|
||||
CLzRef *son;
|
||||
UInt32 matchMaxLen;
|
||||
UInt32 numHashBytes;
|
||||
UInt32 pos;
|
||||
const Byte *buffer;
|
||||
UInt32 cyclicBufferPos;
|
||||
UInt32 cyclicBufferSize; /* it must be historySize + 1 */
|
||||
UInt32 cutValue;
|
||||
|
||||
/* BT + Hash */
|
||||
CMtSync hashSync;
|
||||
/* Byte hashDummy[kMtCacheLineDummy]; */
|
||||
|
||||
/* Hash */
|
||||
Mf_GetHeads GetHeadsFunc;
|
||||
CMatchFinder *MatchFinder;
|
||||
} CMatchFinderMt;
|
||||
|
||||
void MatchFinderMt_Construct(CMatchFinderMt *p);
|
||||
void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAlloc *alloc);
|
||||
SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddBufferBefore,
|
||||
UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAlloc *alloc);
|
||||
void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable);
|
||||
void MatchFinderMt_ReleaseStream(CMatchFinderMt *p);
|
||||
|
||||
EXTERN_C_END
|
||||
|
||||
#endif
|
|
@ -6,7 +6,7 @@
|
|||
/* #include <stdio.h> */
|
||||
#include <string.h>
|
||||
|
||||
/* #define _7ZIP_ST */
|
||||
#define _7ZIP_ST
|
||||
|
||||
#include "Lzma2Enc.h"
|
||||
|
||||
|
|
|
@ -5,6 +5,8 @@
|
|||
|
||||
#include <string.h>
|
||||
|
||||
#define _7ZIP_ST
|
||||
|
||||
/* #define SHOW_STAT */
|
||||
/* #define SHOW_STAT2 */
|
||||
|
||||
|
|
|
@ -1,327 +0,0 @@
|
|||
/* MtCoder.c -- Multi-thread Coder
|
||||
2015-10-13 : Igor Pavlov : Public domain */
|
||||
|
||||
#include "Precomp.h"
|
||||
|
||||
#include "MtCoder.h"
|
||||
|
||||
void LoopThread_Construct(CLoopThread *p)
|
||||
{
|
||||
Thread_Construct(&p->thread);
|
||||
Event_Construct(&p->startEvent);
|
||||
Event_Construct(&p->finishedEvent);
|
||||
}
|
||||
|
||||
void LoopThread_Close(CLoopThread *p)
|
||||
{
|
||||
Thread_Close(&p->thread);
|
||||
Event_Close(&p->startEvent);
|
||||
Event_Close(&p->finishedEvent);
|
||||
}
|
||||
|
||||
static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE LoopThreadFunc(void *pp)
|
||||
{
|
||||
CLoopThread *p = (CLoopThread *)pp;
|
||||
for (;;)
|
||||
{
|
||||
if (Event_Wait(&p->startEvent) != 0)
|
||||
return SZ_ERROR_THREAD;
|
||||
if (p->stop)
|
||||
return 0;
|
||||
p->res = p->func(p->param);
|
||||
if (Event_Set(&p->finishedEvent) != 0)
|
||||
return SZ_ERROR_THREAD;
|
||||
}
|
||||
}
|
||||
|
||||
WRes LoopThread_Create(CLoopThread *p)
|
||||
{
|
||||
p->stop = 0;
|
||||
RINOK(AutoResetEvent_CreateNotSignaled(&p->startEvent));
|
||||
RINOK(AutoResetEvent_CreateNotSignaled(&p->finishedEvent));
|
||||
return Thread_Create(&p->thread, LoopThreadFunc, p);
|
||||
}
|
||||
|
||||
WRes LoopThread_StopAndWait(CLoopThread *p)
|
||||
{
|
||||
p->stop = 1;
|
||||
if (Event_Set(&p->startEvent) != 0)
|
||||
return SZ_ERROR_THREAD;
|
||||
return Thread_Wait(&p->thread);
|
||||
}
|
||||
|
||||
WRes LoopThread_StartSubThread(CLoopThread *p) { return Event_Set(&p->startEvent); }
|
||||
WRes LoopThread_WaitSubThread(CLoopThread *p) { return Event_Wait(&p->finishedEvent); }
|
||||
|
||||
static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize)
|
||||
{
|
||||
return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK;
|
||||
}
|
||||
|
||||
static void MtProgress_Init(CMtProgress *p, ICompressProgress *progress)
|
||||
{
|
||||
unsigned i;
|
||||
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
|
||||
p->inSizes[i] = p->outSizes[i] = 0;
|
||||
p->totalInSize = p->totalOutSize = 0;
|
||||
p->progress = progress;
|
||||
p->res = SZ_OK;
|
||||
}
|
||||
|
||||
static void MtProgress_Reinit(CMtProgress *p, unsigned index)
|
||||
{
|
||||
p->inSizes[index] = 0;
|
||||
p->outSizes[index] = 0;
|
||||
}
|
||||
|
||||
#define UPDATE_PROGRESS(size, prev, total) \
|
||||
if (size != (UInt64)(Int64)-1) { total += size - prev; prev = size; }
|
||||
|
||||
SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize)
|
||||
{
|
||||
SRes res;
|
||||
CriticalSection_Enter(&p->cs);
|
||||
UPDATE_PROGRESS(inSize, p->inSizes[index], p->totalInSize)
|
||||
UPDATE_PROGRESS(outSize, p->outSizes[index], p->totalOutSize)
|
||||
if (p->res == SZ_OK)
|
||||
p->res = Progress(p->progress, p->totalInSize, p->totalOutSize);
|
||||
res = p->res;
|
||||
CriticalSection_Leave(&p->cs);
|
||||
return res;
|
||||
}
|
||||
|
||||
static void MtProgress_SetError(CMtProgress *p, SRes res)
|
||||
{
|
||||
CriticalSection_Enter(&p->cs);
|
||||
if (p->res == SZ_OK)
|
||||
p->res = res;
|
||||
CriticalSection_Leave(&p->cs);
|
||||
}
|
||||
|
||||
static void MtCoder_SetError(CMtCoder* p, SRes res)
|
||||
{
|
||||
CriticalSection_Enter(&p->cs);
|
||||
if (p->res == SZ_OK)
|
||||
p->res = res;
|
||||
CriticalSection_Leave(&p->cs);
|
||||
}
|
||||
|
||||
/* ---------- MtThread ---------- */
|
||||
|
||||
void CMtThread_Construct(CMtThread *p, CMtCoder *mtCoder)
|
||||
{
|
||||
p->mtCoder = mtCoder;
|
||||
p->outBuf = 0;
|
||||
p->inBuf = 0;
|
||||
Event_Construct(&p->canRead);
|
||||
Event_Construct(&p->canWrite);
|
||||
LoopThread_Construct(&p->thread);
|
||||
}
|
||||
|
||||
#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; }
|
||||
|
||||
static void CMtThread_CloseEvents(CMtThread *p)
|
||||
{
|
||||
Event_Close(&p->canRead);
|
||||
Event_Close(&p->canWrite);
|
||||
}
|
||||
|
||||
static void CMtThread_Destruct(CMtThread *p)
|
||||
{
|
||||
CMtThread_CloseEvents(p);
|
||||
|
||||
if (Thread_WasCreated(&p->thread.thread))
|
||||
{
|
||||
LoopThread_StopAndWait(&p->thread);
|
||||
LoopThread_Close(&p->thread);
|
||||
}
|
||||
|
||||
if (p->mtCoder->alloc)
|
||||
IAlloc_Free(p->mtCoder->alloc, p->outBuf);
|
||||
p->outBuf = 0;
|
||||
|
||||
if (p->mtCoder->alloc)
|
||||
IAlloc_Free(p->mtCoder->alloc, p->inBuf);
|
||||
p->inBuf = 0;
|
||||
}
|
||||
|
||||
#define MY_BUF_ALLOC(buf, size, newSize) \
|
||||
if (buf == 0 || size != newSize) \
|
||||
{ IAlloc_Free(p->mtCoder->alloc, buf); \
|
||||
size = newSize; buf = (Byte *)IAlloc_Alloc(p->mtCoder->alloc, size); \
|
||||
if (buf == 0) return SZ_ERROR_MEM; }
|
||||
|
||||
static SRes CMtThread_Prepare(CMtThread *p)
|
||||
{
|
||||
MY_BUF_ALLOC(p->inBuf, p->inBufSize, p->mtCoder->blockSize)
|
||||
MY_BUF_ALLOC(p->outBuf, p->outBufSize, p->mtCoder->destBlockSize)
|
||||
|
||||
p->stopReading = False;
|
||||
p->stopWriting = False;
|
||||
RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canRead));
|
||||
RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canWrite));
|
||||
|
||||
return SZ_OK;
|
||||
}
|
||||
|
||||
static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize)
|
||||
{
|
||||
size_t size = *processedSize;
|
||||
*processedSize = 0;
|
||||
while (size != 0)
|
||||
{
|
||||
size_t curSize = size;
|
||||
SRes res = stream->Read(stream, data, &curSize);
|
||||
*processedSize += curSize;
|
||||
data += curSize;
|
||||
size -= curSize;
|
||||
RINOK(res);
|
||||
if (curSize == 0)
|
||||
return SZ_OK;
|
||||
}
|
||||
return SZ_OK;
|
||||
}
|
||||
|
||||
#define GET_NEXT_THREAD(p) &p->mtCoder->threads[p->index == p->mtCoder->numThreads - 1 ? 0 : p->index + 1]
|
||||
|
||||
static SRes MtThread_Process(CMtThread *p, Bool *stop)
|
||||
{
|
||||
CMtThread *next;
|
||||
*stop = True;
|
||||
if (Event_Wait(&p->canRead) != 0)
|
||||
return SZ_ERROR_THREAD;
|
||||
|
||||
next = GET_NEXT_THREAD(p);
|
||||
|
||||
if (p->stopReading)
|
||||
{
|
||||
next->stopReading = True;
|
||||
return Event_Set(&next->canRead) == 0 ? SZ_OK : SZ_ERROR_THREAD;
|
||||
}
|
||||
|
||||
{
|
||||
size_t size = p->mtCoder->blockSize;
|
||||
size_t destSize = p->outBufSize;
|
||||
|
||||
RINOK(FullRead(p->mtCoder->inStream, p->inBuf, &size));
|
||||
next->stopReading = *stop = (size != p->mtCoder->blockSize);
|
||||
if (Event_Set(&next->canRead) != 0)
|
||||
return SZ_ERROR_THREAD;
|
||||
|
||||
RINOK(p->mtCoder->mtCallback->Code(p->mtCoder->mtCallback, p->index,
|
||||
p->outBuf, &destSize, p->inBuf, size, *stop));
|
||||
|
||||
MtProgress_Reinit(&p->mtCoder->mtProgress, p->index);
|
||||
|
||||
if (Event_Wait(&p->canWrite) != 0)
|
||||
return SZ_ERROR_THREAD;
|
||||
if (p->stopWriting)
|
||||
return SZ_ERROR_FAIL;
|
||||
if (p->mtCoder->outStream->Write(p->mtCoder->outStream, p->outBuf, destSize) != destSize)
|
||||
return SZ_ERROR_WRITE;
|
||||
return Event_Set(&next->canWrite) == 0 ? SZ_OK : SZ_ERROR_THREAD;
|
||||
}
|
||||
}
|
||||
|
||||
static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp)
|
||||
{
|
||||
CMtThread *p = (CMtThread *)pp;
|
||||
for (;;)
|
||||
{
|
||||
Bool stop;
|
||||
CMtThread *next = GET_NEXT_THREAD(p);
|
||||
SRes res = MtThread_Process(p, &stop);
|
||||
if (res != SZ_OK)
|
||||
{
|
||||
MtCoder_SetError(p->mtCoder, res);
|
||||
MtProgress_SetError(&p->mtCoder->mtProgress, res);
|
||||
next->stopReading = True;
|
||||
next->stopWriting = True;
|
||||
Event_Set(&next->canRead);
|
||||
Event_Set(&next->canWrite);
|
||||
return res;
|
||||
}
|
||||
if (stop)
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
void MtCoder_Construct(CMtCoder* p)
|
||||
{
|
||||
unsigned i;
|
||||
p->alloc = 0;
|
||||
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
|
||||
{
|
||||
CMtThread *t = &p->threads[i];
|
||||
t->index = i;
|
||||
CMtThread_Construct(t, p);
|
||||
}
|
||||
CriticalSection_Init(&p->cs);
|
||||
CriticalSection_Init(&p->mtProgress.cs);
|
||||
}
|
||||
|
||||
void MtCoder_Destruct(CMtCoder* p)
|
||||
{
|
||||
unsigned i;
|
||||
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
|
||||
CMtThread_Destruct(&p->threads[i]);
|
||||
CriticalSection_Delete(&p->cs);
|
||||
CriticalSection_Delete(&p->mtProgress.cs);
|
||||
}
|
||||
|
||||
SRes MtCoder_Code(CMtCoder *p)
|
||||
{
|
||||
unsigned i, numThreads = p->numThreads;
|
||||
SRes res = SZ_OK;
|
||||
p->res = SZ_OK;
|
||||
|
||||
MtProgress_Init(&p->mtProgress, p->progress);
|
||||
|
||||
for (i = 0; i < numThreads; i++)
|
||||
{
|
||||
RINOK(CMtThread_Prepare(&p->threads[i]));
|
||||
}
|
||||
|
||||
for (i = 0; i < numThreads; i++)
|
||||
{
|
||||
CMtThread *t = &p->threads[i];
|
||||
CLoopThread *lt = &t->thread;
|
||||
|
||||
if (!Thread_WasCreated(<->thread))
|
||||
{
|
||||
lt->func = ThreadFunc;
|
||||
lt->param = t;
|
||||
|
||||
if (LoopThread_Create(lt) != SZ_OK)
|
||||
{
|
||||
res = SZ_ERROR_THREAD;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (res == SZ_OK)
|
||||
{
|
||||
unsigned j;
|
||||
for (i = 0; i < numThreads; i++)
|
||||
{
|
||||
CMtThread *t = &p->threads[i];
|
||||
if (LoopThread_StartSubThread(&t->thread) != SZ_OK)
|
||||
{
|
||||
res = SZ_ERROR_THREAD;
|
||||
p->threads[0].stopReading = True;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Event_Set(&p->threads[0].canWrite);
|
||||
Event_Set(&p->threads[0].canRead);
|
||||
|
||||
for (j = 0; j < i; j++)
|
||||
LoopThread_WaitSubThread(&p->threads[j].thread);
|
||||
}
|
||||
|
||||
for (i = 0; i < numThreads; i++)
|
||||
CMtThread_CloseEvents(&p->threads[i]);
|
||||
return (res == SZ_OK) ? p->res : res;
|
||||
}
|
|
@ -1,98 +0,0 @@
|
|||
/* MtCoder.h -- Multi-thread Coder
|
||||
2009-11-19 : Igor Pavlov : Public domain */
|
||||
|
||||
#ifndef __MT_CODER_H
|
||||
#define __MT_CODER_H
|
||||
|
||||
#include "Threads.h"
|
||||
|
||||
EXTERN_C_BEGIN
|
||||
|
||||
typedef struct
|
||||
{
|
||||
CThread thread;
|
||||
CAutoResetEvent startEvent;
|
||||
CAutoResetEvent finishedEvent;
|
||||
int stop;
|
||||
|
||||
THREAD_FUNC_TYPE func;
|
||||
LPVOID param;
|
||||
THREAD_FUNC_RET_TYPE res;
|
||||
} CLoopThread;
|
||||
|
||||
void LoopThread_Construct(CLoopThread *p);
|
||||
void LoopThread_Close(CLoopThread *p);
|
||||
WRes LoopThread_Create(CLoopThread *p);
|
||||
WRes LoopThread_StopAndWait(CLoopThread *p);
|
||||
WRes LoopThread_StartSubThread(CLoopThread *p);
|
||||
WRes LoopThread_WaitSubThread(CLoopThread *p);
|
||||
|
||||
#ifndef _7ZIP_ST
|
||||
#define NUM_MT_CODER_THREADS_MAX 32
|
||||
#else
|
||||
#define NUM_MT_CODER_THREADS_MAX 1
|
||||
#endif
|
||||
|
||||
typedef struct
|
||||
{
|
||||
UInt64 totalInSize;
|
||||
UInt64 totalOutSize;
|
||||
ICompressProgress *progress;
|
||||
SRes res;
|
||||
CCriticalSection cs;
|
||||
UInt64 inSizes[NUM_MT_CODER_THREADS_MAX];
|
||||
UInt64 outSizes[NUM_MT_CODER_THREADS_MAX];
|
||||
} CMtProgress;
|
||||
|
||||
SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize);
|
||||
|
||||
struct _CMtCoder;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
struct _CMtCoder *mtCoder;
|
||||
Byte *outBuf;
|
||||
size_t outBufSize;
|
||||
Byte *inBuf;
|
||||
size_t inBufSize;
|
||||
unsigned index;
|
||||
CLoopThread thread;
|
||||
|
||||
Bool stopReading;
|
||||
Bool stopWriting;
|
||||
CAutoResetEvent canRead;
|
||||
CAutoResetEvent canWrite;
|
||||
} CMtThread;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
SRes (*Code)(void *p, unsigned index, Byte *dest, size_t *destSize,
|
||||
const Byte *src, size_t srcSize, int finished);
|
||||
} IMtCoderCallback;
|
||||
|
||||
typedef struct _CMtCoder
|
||||
{
|
||||
size_t blockSize;
|
||||
size_t destBlockSize;
|
||||
unsigned numThreads;
|
||||
|
||||
ISeqInStream *inStream;
|
||||
ISeqOutStream *outStream;
|
||||
ICompressProgress *progress;
|
||||
ISzAlloc *alloc;
|
||||
|
||||
IMtCoderCallback *mtCallback;
|
||||
CCriticalSection cs;
|
||||
SRes res;
|
||||
|
||||
CMtProgress mtProgress;
|
||||
CMtThread threads[NUM_MT_CODER_THREADS_MAX];
|
||||
} CMtCoder;
|
||||
|
||||
void MtCoder_Construct(CMtCoder* p);
|
||||
void MtCoder_Destruct(CMtCoder* p);
|
||||
SRes MtCoder_Code(CMtCoder *p);
|
||||
|
||||
EXTERN_C_END
|
||||
|
||||
#endif
|
|
@ -1,93 +0,0 @@
|
|||
/* Threads.c -- multithreading library
|
||||
2014-09-21 : Igor Pavlov : Public domain */
|
||||
|
||||
#include "Precomp.h"
|
||||
|
||||
#ifndef UNDER_CE
|
||||
#include <process.h>
|
||||
#endif
|
||||
|
||||
#include "Threads.h"
|
||||
|
||||
static WRes GetError()
|
||||
{
|
||||
DWORD res = GetLastError();
|
||||
return (res) ? (WRes)(res) : 1;
|
||||
}
|
||||
|
||||
WRes HandleToWRes(HANDLE h) { return (h != 0) ? 0 : GetError(); }
|
||||
WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); }
|
||||
|
||||
WRes HandlePtr_Close(HANDLE *p)
|
||||
{
|
||||
if (*p != NULL)
|
||||
if (!CloseHandle(*p))
|
||||
return GetError();
|
||||
*p = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
WRes Handle_WaitObject(HANDLE h) { return (WRes)WaitForSingleObject(h, INFINITE); }
|
||||
|
||||
WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param)
|
||||
{
|
||||
/* Windows Me/98/95: threadId parameter may not be NULL in _beginthreadex/CreateThread functions */
|
||||
|
||||
#ifdef UNDER_CE
|
||||
|
||||
DWORD threadId;
|
||||
*p = CreateThread(0, 0, func, param, 0, &threadId);
|
||||
|
||||
#else
|
||||
|
||||
unsigned threadId;
|
||||
*p = (HANDLE)_beginthreadex(NULL, 0, func, param, 0, &threadId);
|
||||
|
||||
#endif
|
||||
|
||||
/* maybe we must use errno here, but probably GetLastError() is also OK. */
|
||||
return HandleToWRes(*p);
|
||||
}
|
||||
|
||||
WRes Event_Create(CEvent *p, BOOL manualReset, int signaled)
|
||||
{
|
||||
*p = CreateEvent(NULL, manualReset, (signaled ? TRUE : FALSE), NULL);
|
||||
return HandleToWRes(*p);
|
||||
}
|
||||
|
||||
WRes Event_Set(CEvent *p) { return BOOLToWRes(SetEvent(*p)); }
|
||||
WRes Event_Reset(CEvent *p) { return BOOLToWRes(ResetEvent(*p)); }
|
||||
|
||||
WRes ManualResetEvent_Create(CManualResetEvent *p, int signaled) { return Event_Create(p, TRUE, signaled); }
|
||||
WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled) { return Event_Create(p, FALSE, signaled); }
|
||||
WRes ManualResetEvent_CreateNotSignaled(CManualResetEvent *p) { return ManualResetEvent_Create(p, 0); }
|
||||
WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p) { return AutoResetEvent_Create(p, 0); }
|
||||
|
||||
|
||||
WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount)
|
||||
{
|
||||
*p = CreateSemaphore(NULL, (LONG)initCount, (LONG)maxCount, NULL);
|
||||
return HandleToWRes(*p);
|
||||
}
|
||||
|
||||
static WRes Semaphore_Release(CSemaphore *p, LONG releaseCount, LONG *previousCount)
|
||||
{ return BOOLToWRes(ReleaseSemaphore(*p, releaseCount, previousCount)); }
|
||||
WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 num)
|
||||
{ return Semaphore_Release(p, (LONG)num, NULL); }
|
||||
WRes Semaphore_Release1(CSemaphore *p) { return Semaphore_ReleaseN(p, 1); }
|
||||
|
||||
WRes CriticalSection_Init(CCriticalSection *p)
|
||||
{
|
||||
/* InitializeCriticalSection can raise only STATUS_NO_MEMORY exception */
|
||||
#ifdef _MSC_VER
|
||||
__try
|
||||
#endif
|
||||
{
|
||||
InitializeCriticalSection(p);
|
||||
/* InitializeCriticalSectionAndSpinCount(p, 0); */
|
||||
}
|
||||
#ifdef _MSC_VER
|
||||
__except (EXCEPTION_EXECUTE_HANDLER) { return 1; }
|
||||
#endif
|
||||
return 0;
|
||||
}
|
|
@ -1,67 +0,0 @@
|
|||
/* Threads.h -- multithreading library
|
||||
2013-11-12 : Igor Pavlov : Public domain */
|
||||
|
||||
#ifndef __7Z_THREADS_H
|
||||
#define __7Z_THREADS_H
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
#include "7zTypes.h"
|
||||
|
||||
EXTERN_C_BEGIN
|
||||
|
||||
WRes HandlePtr_Close(HANDLE *h);
|
||||
WRes Handle_WaitObject(HANDLE h);
|
||||
|
||||
typedef HANDLE CThread;
|
||||
#define Thread_Construct(p) *(p) = NULL
|
||||
#define Thread_WasCreated(p) (*(p) != NULL)
|
||||
#define Thread_Close(p) HandlePtr_Close(p)
|
||||
#define Thread_Wait(p) Handle_WaitObject(*(p))
|
||||
|
||||
typedef
|
||||
#ifdef UNDER_CE
|
||||
DWORD
|
||||
#else
|
||||
unsigned
|
||||
#endif
|
||||
THREAD_FUNC_RET_TYPE;
|
||||
|
||||
#define THREAD_FUNC_CALL_TYPE MY_STD_CALL
|
||||
#define THREAD_FUNC_DECL THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE
|
||||
typedef THREAD_FUNC_RET_TYPE (THREAD_FUNC_CALL_TYPE * THREAD_FUNC_TYPE)(void *);
|
||||
WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param);
|
||||
|
||||
typedef HANDLE CEvent;
|
||||
typedef CEvent CAutoResetEvent;
|
||||
typedef CEvent CManualResetEvent;
|
||||
#define Event_Construct(p) *(p) = NULL
|
||||
#define Event_IsCreated(p) (*(p) != NULL)
|
||||
#define Event_Close(p) HandlePtr_Close(p)
|
||||
#define Event_Wait(p) Handle_WaitObject(*(p))
|
||||
WRes Event_Set(CEvent *p);
|
||||
WRes Event_Reset(CEvent *p);
|
||||
WRes ManualResetEvent_Create(CManualResetEvent *p, int signaled);
|
||||
WRes ManualResetEvent_CreateNotSignaled(CManualResetEvent *p);
|
||||
WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled);
|
||||
WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p);
|
||||
|
||||
typedef HANDLE CSemaphore;
|
||||
#define Semaphore_Construct(p) (*p) = NULL
|
||||
#define Semaphore_Close(p) HandlePtr_Close(p)
|
||||
#define Semaphore_Wait(p) Handle_WaitObject(*(p))
|
||||
WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount);
|
||||
WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 num);
|
||||
WRes Semaphore_Release1(CSemaphore *p);
|
||||
|
||||
typedef CRITICAL_SECTION CCriticalSection;
|
||||
WRes CriticalSection_Init(CCriticalSection *p);
|
||||
#define CriticalSection_Delete(p) DeleteCriticalSection(p)
|
||||
#define CriticalSection_Enter(p) EnterCriticalSection(p)
|
||||
#define CriticalSection_Leave(p) LeaveCriticalSection(p)
|
||||
|
||||
EXTERN_C_END
|
||||
|
||||
#endif
|
Loading…
Reference in a new issue