From f1c7ff63c2312cda7da51bf0eb7f1916e8255aa1 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Mon, 25 May 2020 18:49:49 -0700 Subject: [PATCH 1/2] do not buffer the entire tar file contents to RAM --- extractor.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/extractor.go b/extractor.go index 6bf9085..098aae7 100644 --- a/extractor.go +++ b/extractor.go @@ -100,7 +100,6 @@ func (e *Extractor) Gz(ctx context.Context, body io.Reader, location string, ren // Tar extracts a .tar archived stream of data in the specified location. // It accepts a rename function to handle the names of the files (see the example) func (e *Extractor) Tar(ctx context.Context, body io.Reader, location string, rename Renamer) error { - files := []file{} links := []link{} symlinks := []link{} @@ -145,7 +144,9 @@ func (e *Extractor) Tar(ctx context.Context, body io.Reader, location string, re if _, err := copyCancel(ctx, &data, tr); err != nil { return errors.Annotatef(err, "Read contents of file %s", path) } - files = append(files, file{Path: path, Mode: info.Mode(), Data: data}) + if err := e.copy(ctx, path, info.Mode(), &data); err != nil { + return errors.Annotatef(err, "Create file %s", path) + } case tar.TypeLink: name := header.Linkname if rename != nil { @@ -159,13 +160,7 @@ func (e *Extractor) Tar(ctx context.Context, body io.Reader, location string, re } } - // Now we make another pass creating the files and links - for i := range files { - if err := e.copy(ctx, files[i].Path, files[i].Mode, &files[i].Data); err != nil { - return errors.Annotatef(err, "Create file %s", files[i].Path) - } - } - + // Now we make another pass creating the links for i := range links { select { case <-ctx.Done(): From 431762619bb83ab5b7088e495ea059ac897c1830 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Wed, 27 May 2020 14:16:26 -0700 Subject: [PATCH 2/2] copy from tar to disk directly without reading into memory first --- extractor.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/extractor.go b/extractor.go index 098aae7..a05c8e5 100644 --- a/extractor.go +++ b/extractor.go @@ -140,11 +140,7 @@ func (e *Extractor) Tar(ctx context.Context, body io.Reader, location string, re return errors.Annotatef(err, "Create directory %s", path) } case tar.TypeReg, tar.TypeRegA: - var data bytes.Buffer - if _, err := copyCancel(ctx, &data, tr); err != nil { - return errors.Annotatef(err, "Read contents of file %s", path) - } - if err := e.copy(ctx, path, info.Mode(), &data); err != nil { + if err := e.copy(ctx, path, info.Mode(), tr); err != nil { return errors.Annotatef(err, "Create file %s", path) } case tar.TypeLink: