fix: excessive memory usage

instead of reading entire files before calculating their digests
stream them by using their Reader method.

Signed-off-by: Petu Eusebiu <[email protected]>

pkg/storage/imagestore/imagestore.go

@@ -1,7 +1,6 @@
 package imagestore
 
 import (
-	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/json"
@@ -951,25 +950,25 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, dstDigest godi
 	uuid := u.String()
 	src := is.BlobUploadPath(repo, uuid)
 	digester := sha256.New()
-	buf := new(bytes.Buffer)
 
-	_, err = buf.ReadFrom(body)
+	blobFile, err := is.storeDriver.Writer(src, false)
 	if err != nil {
-		is.log.Error().Err(err).Msg("failed to read blob")
+		is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob")
 
-		return "", -1, err
+		return "", -1, zerr.ErrUploadNotFound
 	}
 
-	nbytes, err := is.storeDriver.WriteFile(src, buf.Bytes())
-	if err != nil {
-		is.log.Error().Err(err).Msg("failed to write blob")
+	defer blobFile.Close()
 
+	mw := io.MultiWriter(blobFile, digester)
+
+	nbytes, err := io.Copy(mw, body)
+	if err != nil {
 		return "", -1, err
 	}
 
-	_, err = digester.Write(buf.Bytes())
-	if err != nil {
-		is.log.Error().Err(err).Str("component", "digester").Msg("failed to write")
+	if err := blobFile.Commit(); err != nil {
+		is.log.Error().Err(err).Str("blob", src).Msg("failed to commit blob")
 
 		return "", -1, err
 	}
@@ -1008,7 +1007,7 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, dstDigest godi
 		}
 	}
 
-	return uuid, int64(nbytes), nil
+	return uuid, nbytes, nil
 }
 
 func (is *ImageStore) DedupeBlob(src string, dstDigest godigest.Digest, dstRepo string, dst string) error {

pkg/storage/scrub.go

@@ -279,21 +279,28 @@ func CheckLayers(
 	}
 
 	for _, layer := range man.Layers {
-		layerContent, err := imgStore.GetBlobContent(imageName, layer.Digest)
+		layerReader, _, err := imgStore.GetBlob(imageName, layer.Digest, "")
 		if err != nil {
 			imageRes = getResult(imageName, tagName, layer.Digest, err)
 
 			break
 		}
 
-		computedDigest := godigest.FromBytes(layerContent)
+		computedDigest, err := godigest.FromReader(layerReader)
+		if err != nil {
+			imageRes = getResult(imageName, tagName, layer.Digest, err)
+
+			break
+		}
 
 		if computedDigest != layer.Digest {
 			imageRes = getResult(imageName, tagName, layer.Digest, errors.ErrBadBlobDigest)
 
 			break
 		}
 
+		_ = layerReader.Close()
+
 		imageRes = getResult(imageName, tagName, "", nil)
 	}