diff --git a/doc/manual/setup.tex b/doc/manual/setup.tex index 8576c038..a1bea7d0 100644 --- a/doc/manual/setup.tex +++ b/doc/manual/setup.tex @@ -478,9 +478,9 @@ \chapter{The setup} sorttype & lowfirst & lowfirst & lowfirst \\ subfilepatches & 64 & 64 & 64 \\ sublargepatches & 64 & 64 & 64 \\ -sublargesize & 4000000 & 4000000 & 4000000 \\ -subsmallextension & 800000 & 800000 & 800000 \\ -subsmallsize & 500000 & 500000 & 500000 \\ +sublargesize & 26880512 & 26880512 & 26880512 \\ +subsmallextension & 3840032 & 3840032 & 3840032 \\ +subsmallsize & 2560016 & 2560016 & 2560016 \\ subsortiosize & 32768 & 32768 & 32768 \\ subtermsinsmall & 10000 & 10000 & 10000 \\ tempdir & . & . & . \\ diff --git a/sources/fsizes.h b/sources/fsizes.h index d0b6f6e1..6fa740ac 100644 --- a/sources/fsizes.h +++ b/sources/fsizes.h @@ -126,10 +126,10 @@ #define MAXFPATCHES 256 #define SORTIOSIZE 200000L -#define SSMALLBUFFER 500000L -#define SSMALLOVERFLOW 800000L +#define SSMALLBUFFER 2560016L +#define SSMALLOVERFLOW 3840032L #define STERMSSMALL 10000L -#define SLARGEBUFFER 4000000L +#define SLARGEBUFFER 26880512L #define SMAXPATCHES 64 #define SMAXFPATCHES 64 #define SSORTIOSIZE 32768L diff --git a/sources/setfile.c b/sources/setfile.c index ecf976e7..cbdcdda0 100644 --- a/sources/setfile.c +++ b/sources/setfile.c @@ -574,6 +574,7 @@ int AllocSetups(VOID) AM.S0 = 0; AM.S0 = AllocSort(LargeSize,SmallSize,SmallEsize,TermsInSmall ,MaxPatches,MaxFpatches,IOsize); + /* AM.S0->file.ziosize was already set to a (larger) value by AllocSort, here it is re-set. */ #ifdef WITHZLIB AM.S0->file.ziosize = IOsize; #ifndef WITHPTHREADS @@ -848,20 +849,27 @@ VOID WriteSetup(VOID) To be used for the main allocation of the sort buffers, and in a later stage for the function and subroutine sort buffers. */ - -SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsInSmall, - int MaxPatches, int MaxFpatches, LONG IOsize) +SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG inTermsInSmall, + int inMaxPatches, int inMaxFpatches, LONG inIOsize) { - LONG allocation,longer,terms2insmall,sortsize,longerp; + LONG LargeSize = inLargeSize; + LONG SmallSize = inSmallSize; + LONG SmallEsize = inSmallEsize; + LONG TermsInSmall = inTermsInSmall; + int MaxPatches = inMaxPatches; + int MaxFpatches = inMaxFpatches; + LONG IOsize = inIOsize; + + LONG longer,terms2insmall,sortsize,longerp; LONG IObuffersize = IOsize; LONG IOtry; SORTING *sort; - int i = 0, j = 0; + int fname2Size = 0, j = 0; char *s; if ( AM.S0 != 0 ) { - s = FG.fname2; i = 0; - while ( *s ) { s++; i++; } - i += 16; + s = FG.fname2; fname2Size = 0; + while ( *s ) { s++; fname2Size++; } + fname2Size += 16; } if ( MaxFpatches < 4 ) MaxFpatches = 4; longer = MaxPatches > MaxFpatches ? MaxPatches : MaxFpatches; @@ -876,7 +884,6 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn terms2insmall = 2*TermsInSmall; /* Used to be just + 100 rather than *2 */ if ( SmallEsize < (SmallSize*3)/2 ) SmallEsize = (SmallSize*3)/2; if ( LargeSize > 0 && LargeSize < 2*SmallSize ) LargeSize = 2*SmallSize; -/* if ( SmallEsize < 3*AM.MaxTer ) SmallEsize = 3*AM.MaxTer; */ SmallEsize = (SmallEsize+15) & (-16L); if ( LargeSize < 0 ) LargeSize = 0; sortsize = sizeof(SORTING); @@ -898,24 +905,24 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn IOtry = ((LargeSize+SmallEsize)/MaxFpatches-2*AM.MaxTer)/sizeof(WORD)-COMPINC; - if ( (LONG)(IObuffersize*sizeof(WORD)) < IOtry ) - IObuffersize = (IOtry+sizeof(WORD)-1)/sizeof(WORD); + /* Here both IObuffersize and IOtry are in units of sizeof(WORD). */ + if ( IObuffersize < IOtry ) { + IObuffersize = IOtry; + } - allocation = - 3*sizeof(POSITION)*(LONG)longer /* Filepositions!! */ - +2*sizeof(WORD *)*longer - +2*(longerp*(sizeof(WORD *)+sizeof(WORD))) - +(3*longerp+2)*sizeof(WORD) -#ifdef WITHZLIB - +(2*longerp+4)*sizeof(WORD) +#if DEBUGGING + if ( LargeSize != inLargeSize ) { MesPrint("Warning: LargeSize adjusted: %l -> %l", inLargeSize, LargeSize); } + if ( SmallSize != inSmallSize ) { MesPrint("Warning: SmallSize adjusted: %l -> %l", inSmallSize, SmallSize); } + if ( SmallEsize != inSmallEsize ) {MesPrint("Warning: SmallEsize adjusted: %l -> %l", inSmallEsize, SmallEsize); } + if ( TermsInSmall != inTermsInSmall ) { MesPrint("Warning: TermsInSmall adjusted: %l -> %l", inTermsInSmall, TermsInSmall); } + if ( MaxPatches != inMaxPatches ) { MesPrint("Warning: MaxPatches adjusted: %d -> %d", inMaxPatches, MaxPatches); } + if ( MaxFpatches != inMaxFpatches ) {MesPrint("Warning: MaxFPatches adjusted: %d -> %d", inMaxFpatches, MaxFpatches); } + /* This one is always changed if the LargeSize has not been... */ + /* if ( IObuffersize != inIOsize/(LONG)sizeof(WORD) ) { MesPrint("Warning: IOsize adjusted: %l -> %l", inIOsize/sizeof(WORD), IObuffersize); } */ #endif - +terms2insmall*sizeof(WORD *) - +terms2insmall*sizeof(WORD *)/2 - +LargeSize - +SmallEsize - +sortsize - +IObuffersize*sizeof(WORD) + i + 16; - sort = (SORTING *)Malloc1(allocation,"sort buffers"); + + /* Allocate separate buffers for most struct members, for better testing and debugging with valgrind. */ + sort = Malloc1(sizeof(*sort), "AllocSort: sorting struct"); sort->LargeSize = LargeSize/sizeof(WORD); sort->SmallSize = SmallSize/sizeof(WORD); @@ -925,33 +932,38 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn sort->TermsInSmall = TermsInSmall; sort->Terms2InSmall = terms2insmall; - sort->sPointer = (WORD **)(sort+1); - sort->SplitScratch = sort->sPointer + terms2insmall; - sort->Patches = (WORD **)(sort->SplitScratch + terms2insmall/2); - sort->pStop = sort->Patches+longer; - sort->poina = sort->pStop+longer; - sort->poin2a = sort->poina + longerp; - sort->fPatches = (POSITION *)(sort->poin2a+longerp); - sort->fPatchesStop = sort->fPatches + longer; - sort->inPatches = sort->fPatchesStop + longer; - sort->tree = (WORD *)(sort->inPatches + longer); - sort->used = sort->tree+longerp; + sort->sPointer = Malloc1(sizeof(*(sort->sPointer ))*terms2insmall, "AllocSort: sPointer"); + sort->Patches = Malloc1(sizeof(*(sort->Patches ))*longer, "AllocSort: Patches"); + sort->pStop = Malloc1(sizeof(*(sort->pStop ))*longer, "AllocSort: pStop"); + sort->poina = Malloc1(sizeof(*(sort->poina ))*longerp, "AllocSort: poina"); + sort->poin2a = Malloc1(sizeof(*(sort->poin2a ))*longerp, "AllocSort: poin2a"); + + sort->fPatches = Malloc1(sizeof(*(sort->fPatches ))*longer, "AllocSort: fPatches"); + sort->fPatchesStop = Malloc1(sizeof(*(sort->fPatchesStop))*longer, "AllocSort: fPatchesStop"); + sort->inPatches = Malloc1(sizeof(*(sort->inPatches ))*longer, "AllocSort: inPatches"); + sort->tree = Malloc1(sizeof(*(sort->tree ))*longerp, "AllocSort: tree"); + sort->used = Malloc1(sizeof(*(sort->used ))*longerp, "AllocSort: used"); + #ifdef WITHZLIB - sort->fpcompressed = sort->used+longerp; - sort->fpincompressed = sort->fpcompressed+longerp+2; - sort->ktoi = sort->fpincompressed+longerp+2; + sort->fpcompressed = Malloc1(sizeof(*(sort->fpcompressed ))*(longerp+2), "AllocSort: fpcompressed"); + sort->fpincompressed = Malloc1(sizeof(*(sort->fpincompressed))*(longerp+2), "AllocSort: fpincompressed"); sort->zsparray = 0; -#else - sort->ktoi = sort->used + longerp; #endif - sort->lBuffer = (WORD *)(sort->ktoi + longerp + 2); + + sort->ktoi = Malloc1(sizeof(*(sort->ktoi))*(longerp+2), "AllocSort: ktoi"); + + // The combined Large buffer and Small buffer (+ extension) are used. + // They must be allocated together. + sort->lBuffer = Malloc1(sizeof(*(sort->lBuffer))*(sort->LargeSize+sort->SmallEsize), "AllocSort: lBuffer+sBuffer"); sort->lTop = sort->lBuffer+sort->LargeSize; + sort->sBuffer = sort->lTop; if ( sort->LargeSize == 0 ) { sort->lBuffer = 0; sort->lTop = 0; } sort->sTop = sort->sBuffer + sort->SmallSize; sort->sTop2 = sort->sBuffer + sort->SmallEsize; sort->sHalf = sort->sBuffer + (LONG)((sort->SmallSize+sort->SmallEsize)>>1); - sort->file.PObuffer = (WORD *)(sort->sTop2); + + sort->file.PObuffer = Malloc1(IObuffersize*sizeof(*(sort->file.PObuffer))+fname2Size+16, "AllocSort: PObuffer"); sort->file.POstop = sort->file.PObuffer+IObuffersize; sort->file.POsize = IObuffersize * sizeof(WORD); sort->file.POfill = sort->file.POfull = sort->file.PObuffer; @@ -962,7 +974,6 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn sort->file.pthreadslock = dummylock; #endif #ifdef WITHZLIB -/* sort->file.ziosize = IOsize; */ sort->file.ziosize = IObuffersize*sizeof(WORD); sort->file.ziobuffer = 0; #endif diff --git a/sources/sort.c b/sources/sort.c index a67c1642..3835d757 100644 --- a/sources/sort.c +++ b/sources/sort.c @@ -4757,7 +4757,22 @@ void CleanUpSort(int num) MUNLOCK(ErrorMessageLock); #endif } - M_free(S,"sorting struct"); + M_free(S->sPointer, "CleanUpSort: sPointer"); + M_free(S->Patches, "CleanUpSort: Patches"); + M_free(S->pStop, "CleanUpSort: pStop"); + M_free(S->poina, "CleanUpSort: poina"); + M_free(S->poin2a, "CleanUpSort: poin2a"); + M_free(S->fPatches, "CleanUpSort: fPatches"); + M_free(S->fPatchesStop, "CleanUpSort: fPatchesStop"); + M_free(S->inPatches, "CleanUpSort: inPatches"); + M_free(S->tree, "CleanUpSort: tree"); + M_free(S->used, "CleanUpSort: used"); + M_free(S->fpcompressed, "CleanUpSort: fpcompressed"); + M_free(S->fpincompressed, "CleanUpSort: fpincompressed"); + M_free(S->ktoi, "CleanUpSort: ktoi"); + M_free(S->lBuffer, "CleanUpSort: lBuffer+sBuffer"); + M_free(S->file.PObuffer, "CleanUpSort: PObuffer"); + M_free(S, "CleanUpSort: sorting struct"); } AN.FunSorts[i] = 0; } diff --git a/sources/structs.h b/sources/structs.h index 5806d132..08c47398 100644 --- a/sources/structs.h +++ b/sources/structs.h @@ -1151,7 +1151,6 @@ typedef struct sOrT { WORD *sFill; /* Filling point in the small buffer */ WORD **sPointer; /* Pointers to terms in the small buffer */ WORD **PoinFill; /* Filling point for pointers to the sm.buf */ - WORD **SplitScratch; /* Excess pointer space for the merge sort */ WORD *cBuffer; /* Compress buffer (if it exists) */ WORD **Patches; /* Positions of patches in large buffer */ WORD **pStop; /* Ends of patches in the large buffer */ @@ -1198,9 +1197,9 @@ typedef struct sOrT { WORD inNum; /* Number of patches on file (input) */ WORD stage4; /* Are we using stage4? */ #ifdef WITHZLIB - PADPOSITION(28,12,12,3,0); + PADPOSITION(27,12,12,3,0); #else - PADPOSITION(25,12,12,3,0); + PADPOSITION(24,12,12,3,0); #endif } SORTING;