Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ require (
github.com/OneOfOne/xxhash v1.2.8
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/adrg/xdg v0.5.3
github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51
github.com/anchore/bubbly v0.0.0-20250717181826-8a411f9d8cbf
github.com/anchore/clio v0.0.0-20250715152405-a0fa658e5084
github.com/anchore/fangs v0.0.0-20250716230140-94c22408c232
Expand Down Expand Up @@ -50,6 +49,7 @@ require (
github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f
github.com/knqyf263/go-deb-version v0.0.0-20241115132648-6f4aee6ccd23
github.com/masahiro331/go-mvn-version v0.0.0-20250131095131-f4974fa13b8a
github.com/mholt/archives v0.1.5
github.com/muesli/termenv v0.16.0
github.com/olekukonko/tablewriter v1.1.1
github.com/openvex/go-vex v0.2.7
Expand Down Expand Up @@ -101,6 +101,7 @@ require (
github.com/STARRY-S/zip v0.2.3 // indirect
github.com/acobaugh/osrelease v0.1.0 // indirect
github.com/agext/levenshtein v1.2.3 // indirect
github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51 // indirect
github.com/anchore/go-lzo v0.1.0 // indirect
github.com/anchore/go-macholibre v0.0.0-20250320151634-807da7ad2331 // indirect
github.com/anchore/go-rpmdb v0.0.0-20250516171929-f77691e1faec // indirect
Expand Down Expand Up @@ -231,7 +232,6 @@ require (
github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 // indirect
github.com/mattn/go-runewidth v0.0.19 // indirect
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
github.com/mholt/archives v0.1.5 // indirect
github.com/mikelolasagasti/xz v1.0.1 // indirect
github.com/minio/minlz v1.0.1 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
Expand Down
56 changes: 54 additions & 2 deletions grype/db/v5/distribution/curator.go
Original file line number Diff line number Diff line change
@@ -1,22 +1,25 @@
package distribution

import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"time"

"github.com/hako/durafmt"
"github.com/hashicorp/go-cleanhttp"
"github.com/mholt/archives"
"github.com/spf13/afero"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"

"github.com/anchore/archiver/v3"
"github.com/anchore/clio"
v5 "github.com/anchore/grype/grype/db/v5"
"github.com/anchore/grype/grype/db/v5/store"
Expand Down Expand Up @@ -358,7 +361,7 @@ func (c *Curator) ImportFrom(dbArchivePath string) error {
return fmt.Errorf("unable to create db temp dir: %w", err)
}

err = archiver.Unarchive(dbArchivePath, tempDir)
err = unarchive(dbArchivePath, tempDir)
if err != nil {
return err
}
Expand Down Expand Up @@ -517,3 +520,52 @@ func defaultHTTPClient(fs afero.Fs, caCertPath string) (*http.Client, error) {
}
return httpClient, nil
}

func unarchive(source, destination string) error {
sourceFile, err := os.Open(source)
if err != nil {
return err
}
defer sourceFile.Close()

format, stream, err := archives.Identify(context.Background(), source, sourceFile)
if err != nil {
return err
}

extractor, ok := format.(archives.Extractor)
if !ok {
return fmt.Errorf("unable to extract DB file, format not supported: %s", source)
}

root, err := os.OpenRoot(destination)
if err != nil {
return err
}

visitor := func(_ context.Context, file archives.FileInfo) error {
if file.IsDir() || file.LinkTarget != "" {
return nil
}

fileReader, err := file.Open()
if err != nil {
return err
}
defer fileReader.Close()

filename := filepath.Clean(file.NameInArchive)

outputFile, err := root.Create(filename)
if err != nil {
return err
}
defer outputFile.Close()

_, err = io.Copy(outputFile, fileReader)

return err
}

return extractor.Extract(context.Background(), stream, visitor)
}
31 changes: 31 additions & 0 deletions grype/db/v5/distribution/curator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"time"

"github.com/gookit/color"
"github.com/mholt/archives"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
Expand Down Expand Up @@ -766,3 +767,33 @@ func TestCurator_Update_setLastSuccessfulUpdateCheck_notCalled(t *testing.T) {
})

}

func Test_unarchive(t *testing.T) {
testFile := filepath.Join(t.TempDir(), "vulnerability.db")
f, err := os.Create(testFile)
require.NoError(t, err)
f.Close()

files, err := archives.FilesFromDisk(t.Context(), nil, map[string]string{
testFile: "",
})
require.NoError(t, err)

source := filepath.Join(t.TempDir(), "archive.tar.zst")
out, err := os.Create(source)
require.NoError(t, err)

format := archives.CompressedArchive{
Compression: archives.Zstd{},
Archival: archives.Tar{},
}
err = format.Archive(t.Context(), out, files)
require.NoError(t, err)

destination := t.TempDir()
err = unarchive(source, destination)
require.NoError(t, err)

expectFile := filepath.Join(destination, "vulnerability.db")
require.FileExists(t, expectFile)
}
55 changes: 53 additions & 2 deletions grype/db/v6/installation/curator.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package installation

import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
Expand All @@ -12,11 +14,11 @@ import (

"github.com/adrg/xdg"
"github.com/hako/durafmt"
"github.com/mholt/archives"
"github.com/spf13/afero"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"

"github.com/anchore/archiver/v3"
"github.com/anchore/clio"
db "github.com/anchore/grype/grype/db/v6"
"github.com/anchore/grype/grype/db/v6/distribution"
Expand Down Expand Up @@ -458,7 +460,7 @@ func (c curator) Import(reference string) error {
} else {
// assume it is an archive
log.Info("unarchiving DB")
err := archiver.Unarchive(reference, tempDir)
err := unarchive(reference, tempDir)
if err != nil {
return err
}
Expand Down Expand Up @@ -615,6 +617,55 @@ func removeAllOrLog(fs afero.Fs, dir string) {
}
}

func unarchive(source, destination string) error {
sourceFile, err := os.Open(source)
if err != nil {
return err
}
defer sourceFile.Close()

format, stream, err := archives.Identify(context.Background(), source, sourceFile)
if err != nil {
return err
}

extractor, ok := format.(archives.Extractor)
if !ok {
return fmt.Errorf("unable to extract DB file, format not supported: %s", source)
}

root, err := os.OpenRoot(destination)
if err != nil {
return err
}

visitor := func(_ context.Context, file archives.FileInfo) error {
if file.IsDir() || file.LinkTarget != "" {
return nil
}

fileReader, err := file.Open()
if err != nil {
return err
}
defer fileReader.Close()

filename := filepath.Clean(file.NameInArchive)

outputFile, err := root.Create(filename)
if err != nil {
return err
}
defer outputFile.Close()

_, err = io.Copy(outputFile, fileReader)

return err
}

return extractor.Extract(context.Background(), stream, visitor)
}

func newMonitor() monitor {
// let consumers know of a monitorable event (download + import stages)
importProgress := progress.NewManual(1)
Expand Down
31 changes: 31 additions & 0 deletions grype/db/v6/installation/curator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"testing"
"time"

"github.com/mholt/archives"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
Expand Down Expand Up @@ -915,6 +916,36 @@ func TestCurator_Import_URL_UsesDBRootDirForDownloadTempBaseAndCleansUp(t *testi
})
}

func Test_unarchive(t *testing.T) {
testFile := filepath.Join(t.TempDir(), "vulnerability.db")
f, err := os.Create(testFile)
require.NoError(t, err)
f.Close()

files, err := archives.FilesFromDisk(t.Context(), nil, map[string]string{
testFile: "",
})
require.NoError(t, err)

source := filepath.Join(t.TempDir(), "archive.tar.zst")
out, err := os.Create(source)
require.NoError(t, err)

format := archives.CompressedArchive{
Compression: archives.Zstd{},
Archival: archives.Tar{},
}
err = format.Archive(t.Context(), out, files)
require.NoError(t, err)

destination := t.TempDir()
err = unarchive(source, destination)
require.NoError(t, err)

expectFile := filepath.Join(destination, "vulnerability.db")
require.FileExists(t, expectFile)
}

func setupTestDB(t *testing.T, dbDir string) db.ReadWriter {
s, err := db.NewWriter(db.Config{
DBDirPath: dbDir,
Expand Down
10 changes: 8 additions & 2 deletions grype/db/v6/testutil/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"crypto/sha256"
"database/sql"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
Expand All @@ -14,9 +15,9 @@ import (
"testing"
"time"

"github.com/mholt/archives"
"github.com/stretchr/testify/require"

"github.com/anchore/archiver/v3"
"github.com/anchore/grype/grype/db/v5/namespace"
distroNs "github.com/anchore/grype/grype/db/v5/namespace/distro"
"github.com/anchore/grype/grype/db/v5/namespace/language"
Expand Down Expand Up @@ -330,7 +331,12 @@ func pack(t *testing.T, typ string, contents []byte) []byte {
require.NoError(t, err)

tarZstd := bytes.Buffer{}
err = archiver.NewZstd().Compress(&tarContents, &tarZstd)
compressor, err := archives.Zstd{}.OpenWriter(&tarZstd)
require.NoError(t, err)

_, err = io.Copy(compressor, &tarContents)
require.NoError(t, err)
err = compressor.Close()
require.NoError(t, err)

return tarZstd.Bytes()
Expand Down
Loading