diff --git a/.github/workflows/update-bootstrap-tools.yml b/.github/workflows/update-bootstrap-tools.yml index 476ff230cd1..8b033dd04ef 100644 --- a/.github/workflows/update-bootstrap-tools.yml +++ b/.github/workflows/update-bootstrap-tools.yml @@ -29,16 +29,16 @@ jobs: run: | make update-tools make list-tools - + export NO_COLOR=1 delimiter="$(openssl rand -hex 8)" - + { echo "status<<${delimiter}" make list-tool-updates echo "${delimiter}" } >> $GITHUB_OUTPUT - + { echo "### Tool version status" echo "\`\`\`" diff --git a/go.mod b/go.mod index ca11c872d70..1268d6b9c69 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/OneOfOne/xxhash v1.2.8 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/adrg/xdg v0.5.3 - github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51 github.com/anchore/bubbly v0.0.0-20250717181826-8a411f9d8cbf github.com/anchore/clio v0.0.0-20250715152405-a0fa658e5084 github.com/anchore/fangs v0.0.0-20250716230140-94c22408c232 @@ -50,6 +49,7 @@ require ( github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f github.com/knqyf263/go-deb-version v0.0.0-20241115132648-6f4aee6ccd23 github.com/masahiro331/go-mvn-version v0.0.0-20250131095131-f4974fa13b8a + github.com/mholt/archives v0.1.5 github.com/muesli/termenv v0.16.0 github.com/olekukonko/tablewriter v1.1.1 github.com/openvex/go-vex v0.2.7 @@ -196,7 +196,6 @@ require ( github.com/goccy/go-yaml v1.18.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect - github.com/golang/snappy v1.0.0 // indirect github.com/google/licensecheck v0.3.1 // indirect github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 // indirect github.com/google/s2a-go v0.1.9 // indirect @@ -231,7 +230,6 @@ require ( github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect - github.com/mholt/archives v0.1.5 // indirect github.com/mikelolasagasti/xz v1.0.1 // indirect github.com/minio/minlz v1.0.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -252,7 +250,6 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 // indirect - github.com/nwaples/rardecode v1.1.3 // indirect github.com/nwaples/rardecode/v2 v2.2.0 // indirect github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/errors v1.1.0 // indirect diff --git a/go.sum b/go.sum index d1af5e8570a..f5ceb08d1c6 100644 --- a/go.sum +++ b/go.sum @@ -130,8 +130,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51 h1:yhk+P8lF3ZiROjmaVRao9WGTRo4b/wYjoKEiAHWrKwc= -github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51/go.mod h1:nwuGSd7aZp0rtYt79YggCGafz1RYsclE7pi3fhLwvuw= github.com/anchore/bubbly v0.0.0-20250717181826-8a411f9d8cbf h1:UY7SQkfVVaeGUpPZrJxqmTc8M0ZSWc5ChiKF6I6aL3I= github.com/anchore/bubbly v0.0.0-20250717181826-8a411f9d8cbf/go.mod h1:w8Br1ZKk1Nk82YRSh10pcD7LO7avPyFmNnaY1TRPgs0= github.com/anchore/clio v0.0.0-20250715152405-a0fa658e5084 h1:7DUAXEdAxoANPlDgxYiaSRKnWnTygvdrrWhnmvEjNLg= @@ -523,8 +521,6 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= -github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -807,8 +803,6 @@ github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 h1:kpt9ZfKcm+ github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1/go.mod h1:qgCw4bBKZX8qMgGeEZzGFVT3notl42dBjNqO2jut0M0= github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE= github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8= -github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= -github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A= github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= diff --git a/grype/db/v5/distribution/curator.go b/grype/db/v5/distribution/curator.go index 609d67a565b..683ffb17457 100644 --- a/grype/db/v5/distribution/curator.go +++ b/grype/db/v5/distribution/curator.go @@ -1,22 +1,25 @@ package distribution import ( + "context" "crypto/tls" "crypto/x509" "fmt" + "io" "net/http" "os" "path" + "path/filepath" "strconv" "time" "github.com/hako/durafmt" "github.com/hashicorp/go-cleanhttp" + "github.com/mholt/archives" "github.com/spf13/afero" "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" - "github.com/anchore/archiver/v3" "github.com/anchore/clio" v5 "github.com/anchore/grype/grype/db/v5" "github.com/anchore/grype/grype/db/v5/store" @@ -358,7 +361,7 @@ func (c *Curator) ImportFrom(dbArchivePath string) error { return fmt.Errorf("unable to create db temp dir: %w", err) } - err = archiver.Unarchive(dbArchivePath, tempDir) + err = unarchive(dbArchivePath, tempDir) if err != nil { return err } @@ -517,3 +520,52 @@ func defaultHTTPClient(fs afero.Fs, caCertPath string) (*http.Client, error) { } return httpClient, nil } + +func unarchive(source, destination string) error { + sourceFile, err := os.Open(source) + if err != nil { + return err + } + defer sourceFile.Close() + + format, stream, err := archives.Identify(context.Background(), source, sourceFile) + if err != nil { + return err + } + + extractor, ok := format.(archives.Extractor) + if !ok { + return fmt.Errorf("unable to extract DB file, format not supported: %s", source) + } + + root, err := os.OpenRoot(destination) + if err != nil { + return err + } + + visitor := func(_ context.Context, file archives.FileInfo) error { + if file.IsDir() || file.LinkTarget != "" { + return nil + } + + fileReader, err := file.Open() + if err != nil { + return err + } + defer fileReader.Close() + + filename := filepath.Clean(file.NameInArchive) + + outputFile, err := root.Create(filename) + if err != nil { + return err + } + defer outputFile.Close() + + _, err = io.Copy(outputFile, fileReader) + + return err + } + + return extractor.Extract(context.Background(), stream, visitor) +} diff --git a/grype/db/v5/distribution/curator_test.go b/grype/db/v5/distribution/curator_test.go index 37b4890cc2d..23e8e4d1f7f 100644 --- a/grype/db/v5/distribution/curator_test.go +++ b/grype/db/v5/distribution/curator_test.go @@ -25,6 +25,7 @@ import ( "time" "github.com/gookit/color" + "github.com/mholt/archives" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -766,3 +767,33 @@ func TestCurator_Update_setLastSuccessfulUpdateCheck_notCalled(t *testing.T) { }) } + +func Test_unarchive(t *testing.T) { + testFile := filepath.Join(t.TempDir(), "vulnerability.db") + f, err := os.Create(testFile) + require.NoError(t, err) + f.Close() + + files, err := archives.FilesFromDisk(t.Context(), nil, map[string]string{ + testFile: "", + }) + require.NoError(t, err) + + source := filepath.Join(t.TempDir(), "archive.tar.zst") + out, err := os.Create(source) + require.NoError(t, err) + + format := archives.CompressedArchive{ + Compression: archives.Zstd{}, + Archival: archives.Tar{}, + } + err = format.Archive(t.Context(), out, files) + require.NoError(t, err) + + destination := t.TempDir() + err = unarchive(source, destination) + require.NoError(t, err) + + expectFile := filepath.Join(destination, "vulnerability.db") + require.FileExists(t, expectFile) +} diff --git a/grype/db/v6/installation/curator.go b/grype/db/v6/installation/curator.go index d52c36cb7b7..c9091bc168b 100644 --- a/grype/db/v6/installation/curator.go +++ b/grype/db/v6/installation/curator.go @@ -1,8 +1,10 @@ package installation import ( + "context" "errors" "fmt" + "io" "os" "path/filepath" "regexp" @@ -12,11 +14,11 @@ import ( "github.com/adrg/xdg" "github.com/hako/durafmt" + "github.com/mholt/archives" "github.com/spf13/afero" "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" - "github.com/anchore/archiver/v3" "github.com/anchore/clio" db "github.com/anchore/grype/grype/db/v6" "github.com/anchore/grype/grype/db/v6/distribution" @@ -458,7 +460,7 @@ func (c curator) Import(reference string) error { } else { // assume it is an archive log.Info("unarchiving DB") - err := archiver.Unarchive(reference, tempDir) + err := unarchive(reference, tempDir) if err != nil { return err } @@ -615,6 +617,55 @@ func removeAllOrLog(fs afero.Fs, dir string) { } } +func unarchive(source, destination string) error { + sourceFile, err := os.Open(source) + if err != nil { + return err + } + defer sourceFile.Close() + + format, stream, err := archives.Identify(context.Background(), source, sourceFile) + if err != nil { + return err + } + + extractor, ok := format.(archives.Extractor) + if !ok { + return fmt.Errorf("unable to extract DB file, format not supported: %s", source) + } + + root, err := os.OpenRoot(destination) + if err != nil { + return err + } + + visitor := func(_ context.Context, file archives.FileInfo) error { + if file.IsDir() || file.LinkTarget != "" { + return nil + } + + fileReader, err := file.Open() + if err != nil { + return err + } + defer fileReader.Close() + + filename := filepath.Clean(file.NameInArchive) + + outputFile, err := root.Create(filename) + if err != nil { + return err + } + defer outputFile.Close() + + _, err = io.Copy(outputFile, fileReader) + + return err + } + + return extractor.Extract(context.Background(), stream, visitor) +} + func newMonitor() monitor { // let consumers know of a monitorable event (download + import stages) importProgress := progress.NewManual(1) diff --git a/grype/db/v6/installation/curator_test.go b/grype/db/v6/installation/curator_test.go index 5ccfbfa4f40..46efb3e8536 100644 --- a/grype/db/v6/installation/curator_test.go +++ b/grype/db/v6/installation/curator_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/mholt/archives" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -915,6 +916,36 @@ func TestCurator_Import_URL_UsesDBRootDirForDownloadTempBaseAndCleansUp(t *testi }) } +func Test_unarchive(t *testing.T) { + testFile := filepath.Join(t.TempDir(), "vulnerability.db") + f, err := os.Create(testFile) + require.NoError(t, err) + f.Close() + + files, err := archives.FilesFromDisk(t.Context(), nil, map[string]string{ + testFile: "", + }) + require.NoError(t, err) + + source := filepath.Join(t.TempDir(), "archive.tar.zst") + out, err := os.Create(source) + require.NoError(t, err) + + format := archives.CompressedArchive{ + Compression: archives.Zstd{}, + Archival: archives.Tar{}, + } + err = format.Archive(t.Context(), out, files) + require.NoError(t, err) + + destination := t.TempDir() + err = unarchive(source, destination) + require.NoError(t, err) + + expectFile := filepath.Join(destination, "vulnerability.db") + require.FileExists(t, expectFile) +} + func setupTestDB(t *testing.T, dbDir string) db.ReadWriter { s, err := db.NewWriter(db.Config{ DBDirPath: dbDir, diff --git a/grype/db/v6/testutil/server.go b/grype/db/v6/testutil/server.go index c8523a4d3bc..d298dddb639 100644 --- a/grype/db/v6/testutil/server.go +++ b/grype/db/v6/testutil/server.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "database/sql" "encoding/json" + "io" "net/http" "net/http/httptest" "os" @@ -14,9 +15,9 @@ import ( "testing" "time" + "github.com/mholt/archives" "github.com/stretchr/testify/require" - "github.com/anchore/archiver/v3" "github.com/anchore/grype/grype/db/v5/namespace" distroNs "github.com/anchore/grype/grype/db/v5/namespace/distro" "github.com/anchore/grype/grype/db/v5/namespace/language" @@ -330,7 +331,12 @@ func pack(t *testing.T, typ string, contents []byte) []byte { require.NoError(t, err) tarZstd := bytes.Buffer{} - err = archiver.NewZstd().Compress(&tarContents, &tarZstd) + compressor, err := archives.Zstd{}.OpenWriter(&tarZstd) + require.NoError(t, err) + + _, err = io.Copy(compressor, &tarContents) + require.NoError(t, err) + err = compressor.Close() require.NoError(t, err) return tarZstd.Bytes()