Skip to content

Reduce memory usage when extracting tar files #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 24, 2020
Merged

Reduce memory usage when extracting tar files #10

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f1c7ff6
do not buffer the entire tar file contents to RAM
tmm1 May 26, 2020
4317626
copy from tar to disk directly without reading into memory first
tmm1 May 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 3 additions & 12 deletions extractor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,6 @@ func (e *Extractor) Gz(ctx context.Context, body io.Reader, location string, ren
// Tar extracts a .tar archived stream of data in the specified location.
// It accepts a rename function to handle the names of the files (see the example)
func (e *Extractor) Tar(ctx context.Context, body io.Reader, location string, rename Renamer) error {
files := []file{}
links := []link{}
symlinks := []link{}

Expand Down Expand Up @@ -141,11 +140,9 @@ func (e *Extractor) Tar(ctx context.Context, body io.Reader, location string, re
return errors.Annotatef(err, "Create directory %s", path)
}
case tar.TypeReg, tar.TypeRegA:
var data bytes.Buffer
if _, err := copyCancel(ctx, &data, tr); err != nil {
return errors.Annotatef(err, "Read contents of file %s", path)
if err := e.copy(ctx, path, info.Mode(), tr); err != nil {
return errors.Annotatef(err, "Create file %s", path)
}
files = append(files, file{Path: path, Mode: info.Mode(), Data: data})
case tar.TypeLink:
name := header.Linkname
if rename != nil {
Expand All @@ -159,13 +156,7 @@ func (e *Extractor) Tar(ctx context.Context, body io.Reader, location string, re
}
}

// Now we make another pass creating the files and links
for i := range files {
if err := e.copy(ctx, files[i].Path, files[i].Mode, &files[i].Data); err != nil {
return errors.Annotatef(err, "Create file %s", files[i].Path)
}
}

// Now we make another pass creating the links
for i := range links {
select {
case <-ctx.Done():
Expand Down