Split the large sort allocation into separate allocations.

Buffer overruns in these allocations will be visible to valgrind.

Author: jodavies

sources/form3.h

@@ -438,6 +438,8 @@ template<typename T> struct calc {
 */
 #define WITHSORTBOTS
 
+#define SPLITALLOC
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>

sources/setfile.c

@@ -848,20 +848,22 @@ VOID WriteSetup(VOID)
 		To be used for the main allocation of the sort buffers, and
 		in a later stage for the function and subroutine sort buffers.
 */
-
 SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsInSmall,
                    int MaxPatches, int MaxFpatches, LONG IOsize)
 {
-	LONG allocation,longer,terms2insmall,sortsize,longerp;
+#ifndef SPLITALLOC
+	LONG allocation;
+#endif
+	LONG longer,terms2insmall,sortsize,longerp;
 	LONG IObuffersize = IOsize;
 	LONG IOtry;
 	SORTING *sort;
-	int i = 0, j = 0;
+	int fname2Size = 0, j = 0;
 	char *s;
 	if ( AM.S0 != 0 ) {
-		s = FG.fname2; i = 0;
-		while ( *s ) { s++; i++; }
-		i += 16;
+		s = FG.fname2; fname2Size = 0;
+		while ( *s ) { s++; fname2Size++; }
+		fname2Size += 16;
 	}
 	if ( MaxFpatches < 4 ) MaxFpatches = 4;
 	longer = MaxPatches > MaxFpatches ? MaxPatches : MaxFpatches;
@@ -897,10 +899,10 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn
 	}
 
 	IOtry = ((LargeSize+SmallEsize)/MaxFpatches-2*AM.MaxTer)/sizeof(WORD)-COMPINC;
-
 	if ( (LONG)(IObuffersize*sizeof(WORD)) < IOtry )
 		IObuffersize = (IOtry+sizeof(WORD)-1)/sizeof(WORD);
 
+#ifndef SPLITALLOC
 	allocation =
 		 3*sizeof(POSITION)*(LONG)longer				/* Filepositions!! */
 		+2*sizeof(WORD *)*longer
@@ -914,7 +916,7 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn
 		+LargeSize
 		+SmallEsize
 		+sortsize
-		+IObuffersize*sizeof(WORD) + i + 16;
+		+IObuffersize*sizeof(WORD) + fname2Size + 16;
 	sort = (SORTING *)Malloc1(allocation,"sort buffers");
 
 	sort->LargeSize = LargeSize/sizeof(WORD);
@@ -931,11 +933,13 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn
 	sort->pStop = sort->Patches+longer;
 	sort->poina = sort->pStop+longer;
 	sort->poin2a = sort->poina + longerp;
+
 	sort->fPatches = (POSITION *)(sort->poin2a+longerp);
 	sort->fPatchesStop = sort->fPatches + longer;
 	sort->inPatches = sort->fPatchesStop + longer;
 	sort->tree = (WORD *)(sort->inPatches + longer);
 	sort->used = sort->tree+longerp;
+
 #ifdef WITHZLIB
 	sort->fpcompressed = sort->used+longerp;
 	sort->fpincompressed = sort->fpcompressed+longerp+2;
@@ -944,13 +948,16 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn
 #else
 	sort->ktoi = sort->used + longerp;
 #endif
+
 	sort->lBuffer = (WORD *)(sort->ktoi + longerp + 2);
 	sort->lTop = sort->lBuffer+sort->LargeSize;
+
 	sort->sBuffer = sort->lTop;
 	if ( sort->LargeSize == 0 ) { sort->lBuffer = 0; sort->lTop = 0; }
 	sort->sTop = sort->sBuffer + sort->SmallSize;
 	sort->sTop2 = sort->sBuffer + sort->SmallEsize;
 	sort->sHalf = sort->sBuffer + (LONG)((sort->SmallSize+sort->SmallEsize)>>1);
+
 	sort->file.PObuffer = (WORD *)(sort->sTop2);
 	sort->file.POstop = sort->file.PObuffer+IObuffersize;
 	sort->file.POsize = IObuffersize * sizeof(WORD);
@@ -975,6 +982,77 @@ SORTING *AllocSort(LONG LargeSize, LONG SmallSize, LONG SmallEsize, LONG TermsIn
 	sort->cBufferSize = 0;
 	sort->f = 0;
 	sort->PolyWise = 0;
+#endif
+
+
+#ifdef SPLITALLOC
+	// Separate buffers.
+	sort = Malloc1(sizeof(*sort), "SPLITALLOC sorting struct");
+
+	sort->LargeSize = LargeSize/sizeof(WORD);
+	sort->SmallSize = SmallSize/sizeof(WORD);
+	sort->SmallEsize = SmallEsize/sizeof(WORD);
+	sort->MaxPatches = MaxPatches;
+	sort->MaxFpatches = MaxFpatches;
+	sort->TermsInSmall = TermsInSmall;
+	sort->Terms2InSmall = terms2insmall;
+
+	sort->sPointer     = Malloc1(sizeof(*(sort->sPointer    ))*terms2insmall, "SPLITALLOC sPointer");
+	sort->SplitScratch = Malloc1(sizeof(*(sort->SplitScratch))*terms2insmall/2, "SPLITALLOC SplitScratch");
+	sort->Patches      = Malloc1(sizeof(*(sort->Patches     ))*longer, "SPLITALLOC Patches");
+	sort->pStop        = Malloc1(sizeof(*(sort->pStop       ))*longer, "SPLITALLOC pStop");
+	sort->poina        = Malloc1(sizeof(*(sort->poina       ))*longerp, "SPLITALLOC poina");
+	sort->poin2a       = Malloc1(sizeof(*(sort->poin2a      ))*longerp, "SPLITALLOC poin2a");
+
+	sort->fPatches     = Malloc1(sizeof(*(sort->fPatches    ))*longer, "SPLITALLOC fPatches");
+	sort->fPatchesStop = Malloc1(sizeof(*(sort->fPatchesStop))*longer, "SPLITALLOC fPatchesStop");
+	sort->inPatches    = Malloc1(sizeof(*(sort->inPatches   ))*longer, "SPLITALLOC inPatches");
+	sort->tree         = Malloc1(sizeof(*(sort->tree        ))*longerp, "SPLITALLOC tree");
+	sort->used         = Malloc1(sizeof(*(sort->used        ))*longerp, "SPLITALLOC used");
+
+#ifdef WITHZLIB
+	sort->fpcompressed   = Malloc1(sizeof(*(sort->fpcompressed  ))*(longerp+2), "SPLITALLOC fpcompressed");
+	sort->fpincompressed = Malloc1(sizeof(*(sort->fpincompressed))*(longerp+2), "SPLITALLOC fpincompressed");
+	sort->zsparray = 0;
+#endif
+
+	sort->ktoi         = Malloc1(sizeof(WORD)*(longerp+2), "SPLITALLOC ktoi");
+
+	// The combined Large buffer and Small buffer (+ extension) are used.
+	// They must be allocated together.
+	sort->lBuffer      = Malloc1(LargeSize+SmallEsize, "SPLITALLOC lBuffer+sBuffer");
+	sort->lTop = sort->lBuffer+sort->LargeSize;
+
+	sort->sBuffer = sort->lTop;
+	if ( sort->LargeSize == 0 ) { sort->lBuffer = 0; sort->lTop = 0; }
+	sort->sTop = sort->sBuffer + sort->SmallSize;
+	sort->sTop2 = sort->sBuffer + sort->SmallEsize;
+	sort->sHalf = sort->sBuffer + (LONG)((sort->SmallSize+sort->SmallEsize)>>1);
+
+	sort->file.PObuffer = Malloc1(IObuffersize*sizeof(*(sort->file.PObuffer))+fname2Size+16, "SPLITALLOC PObuffer");
+	sort->file.POstop = sort->file.PObuffer+IObuffersize;
+	sort->file.POsize = IObuffersize * sizeof(WORD);
+	sort->file.POfill = sort->file.POfull = sort->file.PObuffer;
+	sort->file.active = 0;
+	sort->file.handle = -1;
+	PUTZERO(sort->file.POposition);
+#ifdef WITHPTHREADS
+	sort->file.pthreadslock = dummylock;
+#endif
+#ifdef WITHZLIB
+	sort->file.ziosize = IObuffersize*sizeof(WORD);
+	sort->file.ziobuffer = 0;
+#endif
+	if ( AM.S0 != 0 ) {
+		sort->file.name = (char *)(sort->file.PObuffer + IObuffersize);
+		AllocSortFileName(sort);
+	}
+	else sort->file.name = 0;
+	sort->cBuffer = 0;
+	sort->cBufferSize = 0;
+	sort->f = 0;
+	sort->PolyWise = 0;
+#endif
 
 	return(sort);
 }

sources/sort.c

@@ -4757,7 +4757,27 @@ void CleanUpSort(int num)
 					MUNLOCK(ErrorMessageLock);
 #endif
 				}
+#ifdef SPLITALLOC
+				M_free(S->sPointer, "SPLITALLOC sPointer");
+				M_free(S->SplitScratch, "SPLITALLOC SplitScratch");
+				M_free(S->Patches, "SPLITALLOC Patches");
+				M_free(S->pStop, "SPLITALLOC pStop");
+				M_free(S->poina, "SPLITALLOC poina");
+				M_free(S->poin2a, "SPLITALLOC poin2a");
+				M_free(S->fPatches, "SPLITALLOC fPatches");
+				M_free(S->fPatchesStop, "SPLITALLOC fPatchesStop");
+				M_free(S->inPatches, "SPLITALLOC inPatches");
+				M_free(S->tree, "SPLITALLOC tree");
+				M_free(S->used, "SPLITALLOC used");
+				M_free(S->fpcompressed, "SPLITALLOC fpcompressed");
+				M_free(S->fpincompressed, "SPLITALLOC fpincompressed");
+				M_free(S->ktoi, "SPLITALLOC ktoi");
+				M_free(S->lBuffer, "SPLITALLOC lBuffer+sBuffer");
+				M_free(S->file.PObuffer, "SPLITALLOC PObuffer");
+				M_free(S, "SPLITALLOC sorting struct");
+#else
 				M_free(S,"sorting struct");
+#endif
 			}
 			AN.FunSorts[i] = 0;
 		}