Handle compress edge cases in pyramid chunker

swarm/storage/pyramid.go

@@ -459,6 +459,9 @@ func (pc *PyramidChunker) prepareChunks(isAppend bool) {
 
 		if err != nil {
 			if err == io.EOF || err == io.ErrUnexpectedEOF {
+
+				pc.cleanChunkLevels()
+
 				// Check if we are appending or the chunk is the only one.
 				if parent.branchCount == 1 && (pc.depth() == 0 || isAppend) {
 					// Data is exactly one chunk.. pick the last chunk key as root
@@ -487,6 +490,8 @@ func (pc *PyramidChunker) prepareChunks(isAppend bool) {
 			// Data got exhausted... signal to send any parent tree related chunks
 			if int64(readBytes) < pc.chunkSize {
 
+				pc.cleanChunkLevels()
+
 				// only one data chunk .. so dont add any parent chunk
 				if parent.branchCount <= 1 {
 					chunkWG.Wait()
@@ -622,10 +627,7 @@ func (pc *PyramidChunker) buildTree(isAppend bool, ent *TreeEntry, chunkWG *sync
 		if !isAppend {
 			chunkWG.Wait()
 			if compress {
-				// Remove the chunk level by cutting chunkLevel slice.
-				// Do not set the chunkLevel to nil, as it breaks tree building
-				// in edge cases.
-				pc.chunkLevel = append(pc.chunkLevel[:lvl], append(pc.chunkLevel[lvl+1:], nil)...)
+				pc.chunkLevel[lvl] = nil
 			}
 		}
 	}
@@ -685,3 +687,13 @@ func (pc *PyramidChunker) depth() (d int) {
 	}
 	return
 }
+
+// cleanChunkLevels removes gaps (nil levels) between chunk levels
+// that are not nil.
+func (pc *PyramidChunker) cleanChunkLevels() {
+	for i, l := range pc.chunkLevel {
+		if l == nil {
+			pc.chunkLevel = append(pc.chunkLevel[:i], append(pc.chunkLevel[i+1:], nil)...)
+		}
+	}
+}

swarm/swarm_test.go

@@ -313,7 +313,7 @@ func TestLocalStoreAndRetrieve(t *testing.T) {
 
 	if *longrunning {
 		// test broader set of cases if -longruning flag is set
-		sizes = append(sizes, 83, 179, 253, 1024, 4095, 4096, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678, 67298391, 524288, 524288+4096, 524288+4097, 7*524288, 7*524288+4096, 7*524288+4097, 128*524288, 128*524288+4096, 128*524288+4097)
+		sizes = append(sizes, 83, 179, 253, 1024, 4095, 4096, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678, 67298391, 524288, 524288+4096, 524288+4097, 7*524288, 7*524288+4096, 7*524288+4097, 128*524288, 128*524288+4096, 128*524288+4097, 816778334)
 	}
 	for _, n := range sizes {
 		testLocalStoreAndRetrieve(t, swarm, n, true)