Put pointer-zeroing back into SplitMerge, so GarbHand works

Commit f1b83ae removed various zeroed pointers from SplitMerge, which
does not produce wrong results, since they are "out of range" of the
sorted and merged region of the pointer array, but it leads to GarbHand
computing the total size of the terms incorrectly.

This causes valgrind errors when we write over the end of the SmallExtension,
and causes termination due to a "too small SmallExtension" which might
not actually be the case.

Author: jodavies

check/fixes.frm

@@ -2886,3 +2886,58 @@ EndTerm;
 #pend_if mpi?
 assert runtime_error?("Please increase SubSmallExtension setup parameter.")
 *--#] Issue512_2 :
+*--#[ Issue512_3 :
+#-
+
+* Sort which fits in SmallExtension, but needs GarbHand
+
+#: SmallSize 1000K
+#: SmallExtension 5090K
+#: TermsInSmall 2M
+#: SubSmallSize 1000K
+#: SubSmallExtension 1200K
+#: SubTermsInSmall 2M
+
+Symbol x,n;
+CFunction f,g,prf;
+
+Local test = (<f(1)>+...+<f(150)>)*(<g(1)>+...+<g(100)>);
+.sort
+
+PolyRatFun prf;
+Identify f(x?) = prf(n-x,n+x);
+
+.end
+# Fails due to polynomial size on 32bit builds
+#require wordsize >= 4
+assert succeeded?
+*--#] Issue512_3 :
+*--#[ Issue512_4 :
+#-
+
+* Sort which fits in SubSmallExtension, but needs GarbHand
+
+#: SmallSize 1000K
+#: SmallExtension 1200K
+#: TermsInSmall 2M
+#: SubSmallSize 1000K
+#: SubSmallExtension 5090K
+#: SubTermsInSmall 2M
+
+Symbol x,n;
+CFunction f,g,prf;
+
+Local test = 1;
+.sort
+
+PolyRatFun prf;
+Term;
+	Multiply (<f(1)>+...+<f(150)>)*(<g(1)>+...+<g(100)>);
+	Identify f(x?) = prf(n-x,n+x);
+EndTerm;
+
+.end
+# Fails due to polynomial size on 32bit builds
+#require wordsize >= 4
+assert succeeded?
+*--#] Issue512_4 :

sources/sort.c

@@ -3315,7 +3315,7 @@ LONG SplitMerge(PHEAD WORD **Pointer, LONG number)
 {
 	GETBIDENTITY
 	SORTING *S = AT.SS;
-	WORD **pp3, **pp1, **pp2;
+	WORD **pp3, **pp1, **pp2, **pptop;
 	LONG i, newleft, newright, split;
 
 	if ( number < 2 ) return(number);
@@ -3331,6 +3331,7 @@ LONG SplitMerge(PHEAD WORD **Pointer, LONG number)
 		}
 		return(number);
 	}
+	pptop = Pointer + number;
 	split = number/2;
 	newleft  = SplitMerge(BHEAD Pointer,split);
 	newright = SplitMerge(BHEAD Pointer+split,number-split);
@@ -3354,12 +3355,13 @@ LONG SplitMerge(PHEAD WORD **Pointer, LONG number)
 			if ( AddCoef(BHEAD pp1,pp2) > 0 ) pp1++;
 			else newleft--;
 		  }
-		  pp2++; newright--;
+		  *pp2++ = 0; newright--;
 		}
 		else pp1++;
 		newleft += newright;
 		if ( pp1 < pp2 ) {
 			while ( --newright >= 0 ) *pp1++ = *pp2++;
+			while ( pp1 < pptop ) *pp1++ = 0;
 		}
 		return(newleft);
 	}
@@ -3371,8 +3373,9 @@ LONG SplitMerge(PHEAD WORD **Pointer, LONG number)
 		if ( AN.SplitScratch ) M_free(AN.SplitScratch,"AN.SplitScratch");
 		AN.SplitScratch = (WORD **)Malloc1(AN.SplitScratchSize*sizeof(WORD *),"AN.SplitScratch");
 	}
+
 	pp3 = AN.SplitScratch; pp1 = Pointer;
-	for ( i = 0; i < newleft; i++ ) *pp3++ = *pp1++;
+	for ( i = 0; i < newleft; i++ ) { *pp3++ = *pp1++; }
 	AN.InScratch = newleft;
 	pp1 = AN.SplitScratch; pp2 = Pointer + split; pp3 = Pointer;
 /*
@@ -3396,27 +3399,31 @@ LONG SplitMerge(PHEAD WORD **Pointer, LONG number)
 
 	while ( newleft > 0 && newright > 0 ) {
 		if ( ( i = CompareTerms(BHEAD *pp1,*pp2,(WORD)0) ) < 0 ) {
-			*pp3++ = *pp2++;
+			*pp3++ = *pp2;
+			*pp2++ = 0;
 			newright--;
 		}
 		else if ( i > 0 ) {
-			*pp3++ = *pp1++;
+			*pp3++ = *pp1;
+			*pp1++ = 0;
 			newleft--;
 		}
 		else {
 		  if ( S->PolyWise ) { if ( AddPoly(BHEAD pp1,pp2) > 0 ) *pp3++ = *pp1; }
 		  else {               if ( AddCoef(BHEAD pp1,pp2) > 0 ) *pp3++ = *pp1; }
-		  pp1++; pp2++; newleft--; newright--;
+		  *pp1++ = 0; *pp2++ = 0; newleft--; newright--;
 		}
 	}
-	for ( i = 0; i < newleft; i++ ) *pp3++ = *pp1++;
+	for ( i = 0; i < newleft; i++ ) { *pp3++ = *pp1; *pp1++ = 0; }
 	if ( pp3 == pp2 ) {
 		pp3 += newright;
 	} else {
-		for ( i = 0; i < newright; i++ ) *pp3++ = *pp2++;
+		for ( i = 0; i < newright; i++ ) { *pp3++ = *pp2++; }
 	}
+	newleft = pp3 - Pointer;
+	while ( pp3 < pptop ) *pp3++ = 0;
 	AN.InScratch = 0;
-	return(pp3 - Pointer);
+	return(newleft);
 }
 
 #else
@@ -3545,7 +3552,7 @@ VOID GarbHand(VOID)
 */
 #ifdef TESTGARB
 	MLOCK(ErrorMessageLock);
-	MesPrint("in:  S->sFill = %x, S->sTop2 = %x",S->sFill,S->sTop2);
+	MesPrint("in:  S->sFill = %l, S->sTop2 = %l",S->sFill-S->sBuffer,S->sTop2-S->sBuffer);
 #endif
 	Point = S->sPointer;
 	k = S->sTerms;
@@ -3558,7 +3565,7 @@ VOID GarbHand(VOID)
 		if ( ( s2 = *Point++ ) != 0 ) { total += *s2; }
 	}
 #ifdef TESTGARB
-	MesPrint("total = %l, nterms = %l",2*total,AN.InScratch);
+	MesPrint("total = %l, nterms = %l",total,AN.InScratch);
 	MUNLOCK(ErrorMessageLock);
 #endif
 /*
@@ -3618,7 +3625,7 @@ VOID GarbHand(VOID)
 	S->sFill = s2;
 #ifdef TESTGARB
 	MLOCK(ErrorMessageLock);
-	MesPrint("out: S->sFill = %x, S->sTop2 = %x",S->sFill,S->sTop2);
+	MesPrint("out: S->sFill = %l, S->sTop2 = %l",S->sFill-S->sBuffer,S->sTop2-S->sBuffer);
 	if ( S->sFill >= S->sTop2 ) {
 		MesPrint("We are in deep trouble");
 	}