From e99410e9315cf00aa110185600e130984f2cf7b7 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Mon, 27 Nov 2023 18:59:33 +1300 Subject: [PATCH] feat: add support for `renv.lock` (#668) Resolves #642 --- docs/supported_languages_and_lockfiles.md | 3 +- pkg/lockfile/ecosystems.go | 1 + pkg/lockfile/extract_test.go | 2 + pkg/lockfile/fixtures/renv/empty.lock | 1 + pkg/lockfile/fixtures/renv/not-json.txt | 1 + pkg/lockfile/fixtures/renv/one-package.lock | 17 +++ pkg/lockfile/fixtures/renv/two-packages.lock | 27 ++++ .../fixtures/renv/with-bioconductor.lock | 29 ++++ .../fixtures/renv/with-mixed-sources.lock | 35 +++++ .../fixtures/renv/without-repository.lock | 16 +++ pkg/lockfile/parse-renv-lock.go | 64 +++++++++ pkg/lockfile/parse-renv-lock_test.go | 133 ++++++++++++++++++ pkg/lockfile/parse.go | 1 + pkg/lockfile/parse_test.go | 2 + 14 files changed, 331 insertions(+), 1 deletion(-) create mode 100644 pkg/lockfile/fixtures/renv/empty.lock create mode 100644 pkg/lockfile/fixtures/renv/not-json.txt create mode 100644 pkg/lockfile/fixtures/renv/one-package.lock create mode 100644 pkg/lockfile/fixtures/renv/two-packages.lock create mode 100644 pkg/lockfile/fixtures/renv/with-bioconductor.lock create mode 100644 pkg/lockfile/fixtures/renv/with-mixed-sources.lock create mode 100644 pkg/lockfile/fixtures/renv/without-repository.lock create mode 100644 pkg/lockfile/parse-renv-lock.go create mode 100644 pkg/lockfile/parse-renv-lock_test.go diff --git a/docs/supported_languages_and_lockfiles.md b/docs/supported_languages_and_lockfiles.md index cdc67346ab..f1d8479053 100644 --- a/docs/supported_languages_and_lockfiles.md +++ b/docs/supported_languages_and_lockfiles.md @@ -32,6 +32,7 @@ A wide range of lockfiles are supported by utilizing this [lockfile package](htt | Javascript | `package-lock.json`
`pnpm-lock.yaml`
`yarn.lock`| | PHP | `composer.lock`| | Python | `Pipfile.lock`
`poetry.lock`
`requirements.txt`[\*](https://github.com/google/osv-scanner/issues/34) | +| R | `renv.lock` | | Ruby | `Gemfile.lock`| | Rust | `Cargo.lock`| @@ -102,4 +103,4 @@ Once you extracted your own dependency information, place it in a `osv-scanner.j Then pass this to `osv-scanner` with this: ``` osv-scanner --lockfile osv-scanner:/path/to/osv-scanner.json -``` \ No newline at end of file +``` diff --git a/pkg/lockfile/ecosystems.go b/pkg/lockfile/ecosystems.go index 861373af80..b6dbd96a2c 100644 --- a/pkg/lockfile/ecosystems.go +++ b/pkg/lockfile/ecosystems.go @@ -13,6 +13,7 @@ func KnownEcosystems() []Ecosystem { PipEcosystem, PubEcosystem, ConanEcosystem, + CRANEcosystem, // Disabled temporarily, // see https://github.com/google/osv-scanner/pull/128 discussion for additional context // AlpineEcosystem, diff --git a/pkg/lockfile/extract_test.go b/pkg/lockfile/extract_test.go index b014e1ed6b..647bf3b72d 100644 --- a/pkg/lockfile/extract_test.go +++ b/pkg/lockfile/extract_test.go @@ -48,6 +48,7 @@ func TestFindExtractor(t *testing.T) { "poetry.lock": "poetry.lock", "pom.xml": "pom.xml", "pubspec.lock": "pubspec.lock", + "renv.lock": "renv.lock", "requirements.txt": "requirements.txt", "yarn.lock": "yarn.lock", } @@ -98,6 +99,7 @@ func TestExtractDeps_FindsExpectedExtractor(t *testing.T) { "poetry.lock", "pom.xml", "pubspec.lock", + "renv.lock", "requirements.txt", "yarn.lock", } diff --git a/pkg/lockfile/fixtures/renv/empty.lock b/pkg/lockfile/fixtures/renv/empty.lock new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/pkg/lockfile/fixtures/renv/empty.lock @@ -0,0 +1 @@ +{} diff --git a/pkg/lockfile/fixtures/renv/not-json.txt b/pkg/lockfile/fixtures/renv/not-json.txt new file mode 100644 index 0000000000..3ae3a213d5 --- /dev/null +++ b/pkg/lockfile/fixtures/renv/not-json.txt @@ -0,0 +1 @@ +this is not json! diff --git a/pkg/lockfile/fixtures/renv/one-package.lock b/pkg/lockfile/fixtures/renv/one-package.lock new file mode 100644 index 0000000000..d2156530f7 --- /dev/null +++ b/pkg/lockfile/fixtures/renv/one-package.lock @@ -0,0 +1,17 @@ +{ + "R": { + "Version": "2.15.2", + "Repositories": [] + }, + "Packages": { + "morning": { + "Package": "morning", + "Version": "0.1.0", + "Repository": "CRAN", + "Requirements": [ + "coffee", + "toast" + ] + } + } +} diff --git a/pkg/lockfile/fixtures/renv/two-packages.lock b/pkg/lockfile/fixtures/renv/two-packages.lock new file mode 100644 index 0000000000..a0a425495b --- /dev/null +++ b/pkg/lockfile/fixtures/renv/two-packages.lock @@ -0,0 +1,27 @@ +{ + "R": { + "Version": "4.2.3", + "Repositories": [ + { + "Name": "CRAN", + "URL": "https://cloud.r-project.org" + } + ] + }, + "Packages": { + "markdown": { + "Package": "markdown", + "Version": "1.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "4584a57f565dd7987d59dda3a02cfb41" + }, + "mime": { + "Package": "mime", + "Version": "0.7", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "908d95ccbfd1dd274073ef07a7c93934" + } + } +} diff --git a/pkg/lockfile/fixtures/renv/with-bioconductor.lock b/pkg/lockfile/fixtures/renv/with-bioconductor.lock new file mode 100644 index 0000000000..40bd3105cc --- /dev/null +++ b/pkg/lockfile/fixtures/renv/with-bioconductor.lock @@ -0,0 +1,29 @@ +{ + "R": { + "Version": "4.1.0", + "Repositories": [ + { + "Name": "CRAN", + "URL": "https://cran.rstudio.com" + } + ] + }, + "Bioconductor": { + "Version": "3.13" + }, + "Packages": { + "BH": { + "Package": "BH", + "Version": "1.75.0-0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "e4c04affc2cac20c8fec18385cd14691" + }, + "BSgenome": { + "Package": "BSgenome", + "Version": "1.60.0", + "Source": "Bioconductor", + "Hash": "bc39f66b170caed3ea67c03eb6b4b55c" + } + } +} diff --git a/pkg/lockfile/fixtures/renv/with-mixed-sources.lock b/pkg/lockfile/fixtures/renv/with-mixed-sources.lock new file mode 100644 index 0000000000..6ba0b1bb9f --- /dev/null +++ b/pkg/lockfile/fixtures/renv/with-mixed-sources.lock @@ -0,0 +1,35 @@ +{ + "R": { + "Version": "4.3.1", + "Repositories": [ + { + "Name": "CRAN", + "URL": "https://cloud.r-project.org" + } + ] + }, + "Packages": { + "markdown": { + "Package": "markdown", + "Version": "1.0", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "4584a57f565dd7987d59dda3a02cfb41" + }, + "mime": { + "Package": "mime", + "Version": "0.12.1", + "Source": "GitHub", + "RemoteType": "github", + "RemoteHost": "api.github.com", + "RemoteUsername": "yihui", + "RemoteRepo": "mime", + "RemoteRef": "main", + "RemoteSha": "1763e0dcb72fb58d97bab97bb834fc71f1e012bc", + "Requirements": [ + "tools" + ], + "Hash": "c2772b6269924dad6784aaa1d99dbb86" + } + } +} diff --git a/pkg/lockfile/fixtures/renv/without-repository.lock b/pkg/lockfile/fixtures/renv/without-repository.lock new file mode 100644 index 0000000000..1812d2233f --- /dev/null +++ b/pkg/lockfile/fixtures/renv/without-repository.lock @@ -0,0 +1,16 @@ +{ + "R": { + "Version": "2.15.2", + "Repositories": [] + }, + "Packages": { + "morning": { + "Package": "morning", + "Version": "0.1.0", + "Requirements": [ + "coffee", + "toast" + ] + } + } +} diff --git a/pkg/lockfile/parse-renv-lock.go b/pkg/lockfile/parse-renv-lock.go new file mode 100644 index 0000000000..c398953709 --- /dev/null +++ b/pkg/lockfile/parse-renv-lock.go @@ -0,0 +1,64 @@ +package lockfile + +import ( + "encoding/json" + "fmt" + "path/filepath" +) + +type RenvPackage struct { + Package string `json:"Package"` + Version string `json:"Version"` + Repository string `json:"Repository"` +} + +type RenvLockfile struct { + Packages map[string]RenvPackage `json:"Packages"` +} + +const CRANEcosystem Ecosystem = "CRAN" + +type RenvLockExtractor struct{} + +func (e RenvLockExtractor) ShouldExtract(path string) bool { + return filepath.Base(path) == "renv.lock" +} + +func (e RenvLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { + var parsedLockfile *RenvLockfile + + err := json.NewDecoder(f).Decode(&parsedLockfile) + + if err != nil { + return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err) + } + + packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) + + for _, pkg := range parsedLockfile.Packages { + // currently we only support CRAN + if pkg.Repository != string(CRANEcosystem) { + continue + } + + packages = append(packages, PackageDetails{ + Name: pkg.Package, + Version: pkg.Version, + Ecosystem: CRANEcosystem, + CompareAs: CRANEcosystem, + }) + } + + return packages, nil +} + +var _ Extractor = RenvLockExtractor{} + +//nolint:gochecknoinits +func init() { + registerExtractor("renv.lock", RenvLockExtractor{}) +} + +func ParseRenvLock(pathToLockfile string) ([]PackageDetails, error) { + return extractFromFile(pathToLockfile, RenvLockExtractor{}) +} diff --git a/pkg/lockfile/parse-renv-lock_test.go b/pkg/lockfile/parse-renv-lock_test.go new file mode 100644 index 0000000000..a4f8eecb53 --- /dev/null +++ b/pkg/lockfile/parse-renv-lock_test.go @@ -0,0 +1,133 @@ +package lockfile_test + +import ( + "io/fs" + "testing" + + "github.com/google/osv-scanner/pkg/lockfile" +) + +func TestParseRenvLock_FileDoesNotExist(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseRenvLock("fixtures/renv/does-not-exist") + + expectErrIs(t, err, fs.ErrNotExist) + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParseRenvLock_InvalidJson(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseRenvLock("fixtures/renv/not-json.txt") + + expectErrContaining(t, err, "could not extract from") + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParseRenvLock_NoPackages(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseRenvLock("fixtures/renv/empty.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{}) +} + +func TestParseRenvLock_OnePackage(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseRenvLock("fixtures/renv/one-package.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "morning", + Version: "0.1.0", + Ecosystem: lockfile.CRANEcosystem, + CompareAs: lockfile.CRANEcosystem, + }, + }) +} + +func TestParseRenvLock_TwoPackages(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseRenvLock("fixtures/renv/two-packages.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "markdown", + Version: "1.0", + Ecosystem: lockfile.CRANEcosystem, + CompareAs: lockfile.CRANEcosystem, + }, + { + Name: "mime", + Version: "0.7", + Ecosystem: lockfile.CRANEcosystem, + CompareAs: lockfile.CRANEcosystem, + }, + }) +} + +func TestParseRenvLock_WithMixedSources(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseRenvLock("fixtures/renv/with-mixed-sources.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "markdown", + Version: "1.0", + Ecosystem: lockfile.CRANEcosystem, + CompareAs: lockfile.CRANEcosystem, + }, + }) +} + +func TestParseRenvLock_WithBioconductor(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseRenvLock("fixtures/renv/with-bioconductor.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + // currently Bioconductor is not supported + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "BH", + Version: "1.75.0-0", + Ecosystem: lockfile.CRANEcosystem, + CompareAs: lockfile.CRANEcosystem, + }, + }) +} + +func TestParseRenvLock_WithoutRepository(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseRenvLock("fixtures/renv/without-repository.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{}) +} diff --git a/pkg/lockfile/parse.go b/pkg/lockfile/parse.go index 9e1a80a5bc..5e09984f7a 100644 --- a/pkg/lockfile/parse.go +++ b/pkg/lockfile/parse.go @@ -33,6 +33,7 @@ var parsers = map[string]PackageDetailsParser{ "poetry.lock": ParsePoetryLock, "pom.xml": ParseMavenLock, "pubspec.lock": ParsePubspecLock, + "renv.lock": ParseRenvLock, "requirements.txt": ParseRequirementsTxt, "yarn.lock": ParseYarnLock, } diff --git a/pkg/lockfile/parse_test.go b/pkg/lockfile/parse_test.go index aaef2c1a9f..51f6191d61 100644 --- a/pkg/lockfile/parse_test.go +++ b/pkg/lockfile/parse_test.go @@ -57,6 +57,7 @@ func TestFindParser(t *testing.T) { "poetry.lock", "pom.xml", "pubspec.lock", + "renv.lock", "requirements.txt", "yarn.lock", } @@ -107,6 +108,7 @@ func TestParse_FindsExpectedParsers(t *testing.T) { "poetry.lock", "pom.xml", "pubspec.lock", + "renv.lock", "requirements.txt", "yarn.lock", }