From bf33ac89177a0f8960ccaa9ee251f6b068f40579 Mon Sep 17 00:00:00 2001 From: Colm O hEigeartaigh Date: Mon, 20 Nov 2023 07:06:40 +0000 Subject: [PATCH 1/6] Adding the ability to retrieve remote licenses for yarn.lock Signed-off-by: Colm O hEigeartaigh --- cmd/syft/cli/options/catalog.go | 4 ++ cmd/syft/cli/options/javascript.go | 5 ++ syft/pkg/cataloger/cataloger.go | 4 +- syft/pkg/cataloger/config.go | 2 + syft/pkg/cataloger/javascript/cataloger.go | 8 ++- .../cataloger/javascript/cataloger_test.go | 4 +- syft/pkg/cataloger/javascript/options.go | 18 ++++++ syft/pkg/cataloger/javascript/package.go | 60 ++++++++++++++++++- .../cataloger/javascript/parse_yarn_lock.go | 13 ++-- .../javascript/parse_yarn_lock_test.go | 10 +++- 10 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 cmd/syft/cli/options/javascript.go create mode 100644 syft/pkg/cataloger/javascript/options.go diff --git a/cmd/syft/cli/options/catalog.go b/cmd/syft/cli/options/catalog.go index d8b9f8e59bd..9fc4a7496c8 100644 --- a/cmd/syft/cli/options/catalog.go +++ b/cmd/syft/cli/options/catalog.go @@ -15,6 +15,7 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger" golangCataloger "github.com/anchore/syft/syft/pkg/cataloger/golang" javaCataloger "github.com/anchore/syft/syft/pkg/cataloger/java" + javascriptCataloger "github.com/anchore/syft/syft/pkg/cataloger/javascript" "github.com/anchore/syft/syft/pkg/cataloger/kernel" pythonCataloger "github.com/anchore/syft/syft/pkg/cataloger/python" "github.com/anchore/syft/syft/source" @@ -25,6 +26,7 @@ type Catalog struct { Package pkg `yaml:"package" json:"package" mapstructure:"package"` Golang golang `yaml:"golang" json:"golang" mapstructure:"golang"` Java java `yaml:"java" json:"java" mapstructure:"java"` + Javascript javascript `yaml:"javascript" json:"javascript" mapstructure:"javascript"` LinuxKernel linuxKernel `yaml:"linux-kernel" json:"linux-kernel" mapstructure:"linux-kernel"` Python python `yaml:"python" json:"python" mapstructure:"python"` FileMetadata fileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"` @@ -143,6 +145,8 @@ func (cfg Catalog) ToCatalogerConfig() cataloger.Config { WithUseNetwork(cfg.Java.UseNetwork). WithMavenURL(cfg.Java.MavenURL). WithMaxParentRecursiveDepth(cfg.Java.MaxParentRecursiveDepth), + Javascript: javascriptCataloger.NewCatalogerOpts(). + WithSearchRemoteLicenses(cfg.Javascript.SearchRemoteLicenses), Python: pythonCataloger.CatalogerConfig{ GuessUnpinnedRequirements: cfg.Python.GuessUnpinnedRequirements, }, diff --git a/cmd/syft/cli/options/javascript.go b/cmd/syft/cli/options/javascript.go new file mode 100644 index 00000000000..094b4a836e2 --- /dev/null +++ b/cmd/syft/cli/options/javascript.go @@ -0,0 +1,5 @@ +package options + +type javascript struct { + SearchRemoteLicenses bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"` +} diff --git a/syft/pkg/cataloger/cataloger.go b/syft/pkg/cataloger/cataloger.go index a66a0569eb6..b9c4867cda5 100644 --- a/syft/pkg/cataloger/cataloger.go +++ b/syft/pkg/cataloger/cataloger.go @@ -85,7 +85,7 @@ func DirectoryCatalogers(cfg Config) []pkg.Cataloger { java.NewGradleLockfileCataloger(), java.NewPomCataloger(), java.NewNativeImageCataloger(), - javascript.NewLockCataloger(), + javascript.NewLockCataloger(cfg.Javascript), nix.NewStoreCataloger(), php.NewComposerLockCataloger(), gentoo.NewPortageCataloger(), @@ -124,7 +124,7 @@ func AllCatalogers(cfg Config) []pkg.Cataloger { java.NewGradleLockfileCataloger(), java.NewPomCataloger(), java.NewNativeImageCataloger(), - javascript.NewLockCataloger(), + javascript.NewLockCataloger(cfg.Javascript), javascript.NewPackageCataloger(), kernel.NewLinuxKernelCataloger(cfg.LinuxKernel), nix.NewStoreCataloger(), diff --git a/syft/pkg/cataloger/config.go b/syft/pkg/cataloger/config.go index 3eced5e0f95..9c38c2d7c14 100644 --- a/syft/pkg/cataloger/config.go +++ b/syft/pkg/cataloger/config.go @@ -3,6 +3,7 @@ package cataloger import ( "github.com/anchore/syft/syft/pkg/cataloger/golang" "github.com/anchore/syft/syft/pkg/cataloger/java" + "github.com/anchore/syft/syft/pkg/cataloger/javascript" "github.com/anchore/syft/syft/pkg/cataloger/kernel" "github.com/anchore/syft/syft/pkg/cataloger/python" ) @@ -14,6 +15,7 @@ type Config struct { LinuxKernel kernel.LinuxCatalogerConfig Python python.CatalogerConfig Java java.CatalogerOpts + Javascript javascript.CatalogerOpts Catalogers []string Parallelism int ExcludeBinaryOverlapByOwnership bool diff --git a/syft/pkg/cataloger/javascript/cataloger.go b/syft/pkg/cataloger/javascript/cataloger.go index 56127e373cd..a4ee7f4e0f1 100644 --- a/syft/pkg/cataloger/javascript/cataloger.go +++ b/syft/pkg/cataloger/javascript/cataloger.go @@ -14,9 +14,13 @@ func NewPackageCataloger() *generic.Cataloger { } // NewLockCataloger returns a new cataloger object for NPM (and NPM-adjacent, such as yarn) lock files. -func NewLockCataloger() *generic.Cataloger { +func NewLockCataloger(opts CatalogerOpts) *generic.Cataloger { + c := yarnCataloger{ + opts: opts, + } + return generic.NewCataloger("javascript-lock-cataloger"). WithParserByGlobs(parsePackageLock, "**/package-lock.json"). - WithParserByGlobs(parseYarnLock, "**/yarn.lock"). + WithParserByGlobs(c.parseYarnLock, "**/yarn.lock"). WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml") } diff --git a/syft/pkg/cataloger/javascript/cataloger_test.go b/syft/pkg/cataloger/javascript/cataloger_test.go index c4db1754596..5463e64ce48 100644 --- a/syft/pkg/cataloger/javascript/cataloger_test.go +++ b/syft/pkg/cataloger/javascript/cataloger_test.go @@ -132,7 +132,7 @@ func Test_JavascriptCataloger(t *testing.T) { pkgtest.NewCatalogTester(). FromDirectory(t, "test-fixtures/pkg-lock"). Expects(expectedPkgs, nil). - TestCataloger(t, NewLockCataloger()) + TestCataloger(t, NewLockCataloger(CatalogerOpts{})) } @@ -183,7 +183,7 @@ func Test_LockCataloger_Globs(t *testing.T) { pkgtest.NewCatalogTester(). FromDirectory(t, test.fixture). ExpectsResolverContentQueries(test.expected). - TestCataloger(t, NewLockCataloger()) + TestCataloger(t, NewLockCataloger(CatalogerOpts{})) }) } } diff --git a/syft/pkg/cataloger/javascript/options.go b/syft/pkg/cataloger/javascript/options.go new file mode 100644 index 00000000000..cea99f6bbbe --- /dev/null +++ b/syft/pkg/cataloger/javascript/options.go @@ -0,0 +1,18 @@ +package javascript + +type CatalogerOpts struct { + searchRemoteLicenses bool +} + +func (g CatalogerOpts) WithSearchRemoteLicenses(input bool) CatalogerOpts { + g.searchRemoteLicenses = input + return g +} + +// NewCatalogerOpts create a NewCatalogerOpts with default options, which includes: +// - searchRemoteLicenses is false +func NewCatalogerOpts() CatalogerOpts { + g := CatalogerOpts{} + + return g +} diff --git a/syft/pkg/cataloger/javascript/package.go b/syft/pkg/cataloger/javascript/package.go index 9a1331ac6e6..aba54898237 100644 --- a/syft/pkg/cataloger/javascript/package.go +++ b/syft/pkg/cataloger/javascript/package.go @@ -2,9 +2,12 @@ package javascript import ( "encoding/json" + "fmt" "io" + "net/http" "path" "strings" + "time" "github.com/anchore/packageurl-go" "github.com/anchore/syft/internal/log" @@ -106,13 +109,24 @@ func newPnpmPackage(resolver file.Resolver, location file.Location, name, versio ) } -func newYarnLockPackage(resolver file.Resolver, location file.Location, name, version string) pkg.Package { +func newYarnLockPackage(searchRemoteLicenses bool, resolver file.Resolver, location file.Location, name, version string) pkg.Package { + var licenseSet pkg.LicenseSet + + if searchRemoteLicenses { + license, err := getLicenseFromNpmRegistry(name, version) + if err == nil && license != "" { + licenses := pkg.NewLicensesFromValues(license) + licenseSet = pkg.NewLicenseSet(licenses...) + } + } + return finalizeLockPkg( resolver, location, pkg.Package{ Name: name, Version: version, + Licenses: licenseSet, Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), PURL: packageURL(name, version), Language: pkg.JavaScript, @@ -121,6 +135,50 @@ func newYarnLockPackage(resolver file.Resolver, location file.Location, name, ve ) } +func getLicenseFromNpmRegistry(packageName, version string) (string, error) { + var requestURL = fmt.Sprintf("https://registry.npmjs.org/%s/%s", packageName, version) + log.Tracef("trying to fetch remote package %s", requestURL) + + npmRequest, err := http.NewRequest(http.MethodGet, requestURL, nil) + if err != nil { + return "", fmt.Errorf("unable to format remote request: %w", err) + } + + httpClient := &http.Client{ + Timeout: time.Second * 10, + } + + resp, err := httpClient.Do(npmRequest) + if err != nil { + return "", fmt.Errorf("unable to get package from npm registry: %w", err) + } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Errorf("unable to close body: %+v", err) + } + }() + + bytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("unable to parse package from npm registry: %w", err) + } + + dec := json.NewDecoder(strings.NewReader(string(bytes))) + + // Read "license" from the response + var license struct { + License string `json:"license"` + } + + if err := dec.Decode(&license); err != nil { + return "", fmt.Errorf("unable to parse license from npm registry: %w", err) + } + + log.Tracef("Retrieved License: %s", license.License) + + return license.License, nil +} + func finalizeLockPkg(resolver file.Resolver, location file.Location, p pkg.Package) pkg.Package { licenseCandidate := addLicenses(p.Name, resolver, location) p.Licenses.Add(pkg.NewLicensesFromLocation(location, licenseCandidate...)...) diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index d42490ed30d..0d655553e44 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -13,8 +13,9 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/generic" ) -// integrity check -var _ generic.Parser = parseYarnLock +type yarnCataloger struct { + opts CatalogerOpts +} var ( // packageNameExp matches the name of the dependency in yarn.lock @@ -43,7 +44,7 @@ const ( noVersion = "" ) -func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { +func (c *yarnCataloger) parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { // in the case we find yarn.lock files in the node_modules directories, skip those // as the whole purpose of the lock file is for the specific dependencies of the project if pathContainsNodeModulesDirectory(reader.Path()) { @@ -62,7 +63,7 @@ func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.L if packageName := findPackageName(line); packageName != noPackage { // When we find a new package, check if we have unsaved identifiers if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) { - pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, currentPackage, currentVersion)) + pkgs = append(pkgs, newYarnLockPackage(c.opts.searchRemoteLicenses, resolver, reader.Location, currentPackage, currentVersion)) parsedPackages.Add(currentPackage + "@" + currentVersion) } @@ -70,7 +71,7 @@ func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.L } else if version := findPackageVersion(line); version != noVersion { currentVersion = version } else if packageName, version := findPackageAndVersion(line); packageName != noPackage && version != noVersion && !parsedPackages.Has(packageName+"@"+version) { - pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, packageName, version)) + pkgs = append(pkgs, newYarnLockPackage(c.opts.searchRemoteLicenses, resolver, reader.Location, packageName, version)) parsedPackages.Add(packageName + "@" + version) // Cleanup to indicate no unsaved identifiers @@ -81,7 +82,7 @@ func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.L // check if we have valid unsaved data after end-of-file has reached if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) { - pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, currentPackage, currentVersion)) + pkgs = append(pkgs, newYarnLockPackage(c.opts.searchRemoteLicenses, resolver, reader.Location, currentPackage, currentVersion)) parsedPackages.Add(currentPackage + "@" + currentVersion) } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go index cb2dacc407c..9bad9b14e32 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go @@ -91,7 +91,10 @@ func TestParseYarnBerry(t *testing.T) { }, } - pkgtest.TestFileParser(t, fixture, parseYarnLock, expectedPkgs, expectedRelationships) + c := yarnCataloger{ + opts: CatalogerOpts{}, + } + pkgtest.TestFileParser(t, fixture, c.parseYarnLock, expectedPkgs, expectedRelationships) } @@ -177,7 +180,10 @@ func TestParseYarnLock(t *testing.T) { }, } - pkgtest.TestFileParser(t, fixture, parseYarnLock, expectedPkgs, expectedRelationships) + c := yarnCataloger{ + opts: CatalogerOpts{}, + } + pkgtest.TestFileParser(t, fixture, c.parseYarnLock, expectedPkgs, expectedRelationships) } From d12bd90921c1142e028fcdc9c26e7d9cc35325d7 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Tue, 5 Dec 2023 10:40:51 -0500 Subject: [PATCH 2/6] fix: update back to adapter pattern after merge Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/config.go | 2 +- syft/pkg/cataloger/javascript/cataloger.go | 5 +++-- .../pkg/cataloger/javascript/cataloger_test.go | 4 ++-- syft/pkg/cataloger/javascript/options.go | 8 ++++---- .../cataloger/javascript/parse_yarn_lock.go | 18 ++++++++++++++---- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/syft/pkg/cataloger/config.go b/syft/pkg/cataloger/config.go index f7df5a8f883..157a2572926 100644 --- a/syft/pkg/cataloger/config.go +++ b/syft/pkg/cataloger/config.go @@ -16,7 +16,7 @@ type Config struct { LinuxKernel kernel.LinuxKernelCatalogerConfig Python python.CatalogerConfig Java java.ArchiveCatalogerConfig - Javascript javascript.CatalogerConfig + Javascript javascript.YarnLockCatalogerConfig Catalogers []string Parallelism int ExcludeBinaryOverlapByOwnership bool diff --git a/syft/pkg/cataloger/javascript/cataloger.go b/syft/pkg/cataloger/javascript/cataloger.go index ca18ae7465d..84208ed69bc 100644 --- a/syft/pkg/cataloger/javascript/cataloger.go +++ b/syft/pkg/cataloger/javascript/cataloger.go @@ -15,9 +15,10 @@ func NewPackageCataloger() pkg.Cataloger { } // NewLockCataloger returns a new cataloger object for NPM (and NPM-adjacent, such as yarn) lock files. -func NewLockCataloger(_ CatalogerConfig) pkg.Cataloger { +func NewLockCataloger(cfg YarnLockCatalogerConfig) pkg.Cataloger { + yarnLockAdapter := newGenericYarnLockAdapter(cfg) return generic.NewCataloger("javascript-lock-cataloger"). WithParserByGlobs(parsePackageLock, "**/package-lock.json"). - WithParserByGlobs(parseYarnLock, "**/yarn.lock"). + WithParserByGlobs(yarnLockAdapter.parseYarnLock, "**/yarn.lock"). WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml") } diff --git a/syft/pkg/cataloger/javascript/cataloger_test.go b/syft/pkg/cataloger/javascript/cataloger_test.go index 0adf2b4fd62..77d32e2fba7 100644 --- a/syft/pkg/cataloger/javascript/cataloger_test.go +++ b/syft/pkg/cataloger/javascript/cataloger_test.go @@ -132,7 +132,7 @@ func Test_JavascriptCataloger(t *testing.T) { pkgtest.NewCatalogTester(). FromDirectory(t, "test-fixtures/pkg-lock"). Expects(expectedPkgs, nil). - TestCataloger(t, NewLockCataloger(CatalogerConfig{})) + TestCataloger(t, NewLockCataloger(YarnLockCatalogerConfig{})) } @@ -183,7 +183,7 @@ func Test_LockCataloger_Globs(t *testing.T) { pkgtest.NewCatalogTester(). FromDirectory(t, test.fixture). ExpectsResolverContentQueries(test.expected). - TestCataloger(t, NewLockCataloger(CatalogerConfig{})) + TestCataloger(t, NewLockCataloger(YarnLockCatalogerConfig{})) }) } } diff --git a/syft/pkg/cataloger/javascript/options.go b/syft/pkg/cataloger/javascript/options.go index cbadaa4ab22..2bf0078f3a4 100644 --- a/syft/pkg/cataloger/javascript/options.go +++ b/syft/pkg/cataloger/javascript/options.go @@ -1,18 +1,18 @@ package javascript -type CatalogerConfig struct { +type YarnLockCatalogerConfig struct { searchRemoteLicenses bool } -func (g CatalogerConfig) WithSearchRemoteLicenses(input bool) CatalogerConfig { +func (g YarnLockCatalogerConfig) WithSearchRemoteLicenses(input bool) YarnLockCatalogerConfig { g.searchRemoteLicenses = input return g } // NewCatalogerOpts create a NewCatalogerOpts with default options, which includes: // - searchRemoteLicenses is false -func NewCatalogerOpts() CatalogerConfig { - g := CatalogerConfig{} +func NewCatalogerOpts() YarnLockCatalogerConfig { + g := YarnLockCatalogerConfig{} return g } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index b38740983c4..285d2ac0bf4 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -43,7 +43,17 @@ const ( noVersion = "" ) -func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { +type genericYarnLockAdapter struct { + cfg YarnLockCatalogerConfig +} + +func newGenericYarnLockAdapter(cfg YarnLockCatalogerConfig) genericYarnLockAdapter { + return genericYarnLockAdapter{ + cfg: cfg, + } +} + +func (a genericYarnLockAdapter) parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { // in the case we find yarn.lock files in the node_modules directories, skip those // as the whole purpose of the lock file is for the specific dependencies of the project if pathContainsNodeModulesDirectory(reader.Path()) { @@ -62,7 +72,7 @@ func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.L if packageName := findPackageName(line); packageName != noPackage { // When we find a new package, check if we have unsaved identifiers if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) { - pkgs = append(pkgs, newYarnLockPackage(false, resolver, reader.Location, currentPackage, currentVersion)) + pkgs = append(pkgs, newYarnLockPackage(a.cfg.searchRemoteLicenses, resolver, reader.Location, currentPackage, currentVersion)) parsedPackages.Add(currentPackage + "@" + currentVersion) } @@ -70,7 +80,7 @@ func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.L } else if version := findPackageVersion(line); version != noVersion { currentVersion = version } else if packageName, version := findPackageAndVersion(line); packageName != noPackage && version != noVersion && !parsedPackages.Has(packageName+"@"+version) { - pkgs = append(pkgs, newYarnLockPackage(false, resolver, reader.Location, packageName, version)) + pkgs = append(pkgs, newYarnLockPackage(a.cfg.searchRemoteLicenses, resolver, reader.Location, packageName, version)) parsedPackages.Add(packageName + "@" + version) // Cleanup to indicate no unsaved identifiers @@ -81,7 +91,7 @@ func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.L // check if we have valid unsaved data after end-of-file has reached if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) { - pkgs = append(pkgs, newYarnLockPackage(false, resolver, reader.Location, currentPackage, currentVersion)) + pkgs = append(pkgs, newYarnLockPackage(a.cfg.searchRemoteLicenses, resolver, reader.Location, currentPackage, currentVersion)) parsedPackages.Add(currentPackage + "@" + currentVersion) } From 3e227aee6273a4b90c0037d67bcc9c4c59d1b373 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Tue, 5 Dec 2023 10:46:55 -0500 Subject: [PATCH 3/6] test: change tests back to use adapter Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/config.go | 2 +- syft/pkg/cataloger/javascript/cataloger.go | 2 +- syft/pkg/cataloger/javascript/cataloger_test.go | 4 ++-- syft/pkg/cataloger/javascript/options.go | 8 ++++---- syft/pkg/cataloger/javascript/parse_yarn_lock.go | 7 ++----- syft/pkg/cataloger/javascript/parse_yarn_lock_test.go | 6 ++++-- 6 files changed, 14 insertions(+), 15 deletions(-) diff --git a/syft/pkg/cataloger/config.go b/syft/pkg/cataloger/config.go index 157a2572926..f7df5a8f883 100644 --- a/syft/pkg/cataloger/config.go +++ b/syft/pkg/cataloger/config.go @@ -16,7 +16,7 @@ type Config struct { LinuxKernel kernel.LinuxKernelCatalogerConfig Python python.CatalogerConfig Java java.ArchiveCatalogerConfig - Javascript javascript.YarnLockCatalogerConfig + Javascript javascript.CatalogerConfig Catalogers []string Parallelism int ExcludeBinaryOverlapByOwnership bool diff --git a/syft/pkg/cataloger/javascript/cataloger.go b/syft/pkg/cataloger/javascript/cataloger.go index 84208ed69bc..256c759da70 100644 --- a/syft/pkg/cataloger/javascript/cataloger.go +++ b/syft/pkg/cataloger/javascript/cataloger.go @@ -15,7 +15,7 @@ func NewPackageCataloger() pkg.Cataloger { } // NewLockCataloger returns a new cataloger object for NPM (and NPM-adjacent, such as yarn) lock files. -func NewLockCataloger(cfg YarnLockCatalogerConfig) pkg.Cataloger { +func NewLockCataloger(cfg CatalogerConfig) pkg.Cataloger { yarnLockAdapter := newGenericYarnLockAdapter(cfg) return generic.NewCataloger("javascript-lock-cataloger"). WithParserByGlobs(parsePackageLock, "**/package-lock.json"). diff --git a/syft/pkg/cataloger/javascript/cataloger_test.go b/syft/pkg/cataloger/javascript/cataloger_test.go index 77d32e2fba7..0adf2b4fd62 100644 --- a/syft/pkg/cataloger/javascript/cataloger_test.go +++ b/syft/pkg/cataloger/javascript/cataloger_test.go @@ -132,7 +132,7 @@ func Test_JavascriptCataloger(t *testing.T) { pkgtest.NewCatalogTester(). FromDirectory(t, "test-fixtures/pkg-lock"). Expects(expectedPkgs, nil). - TestCataloger(t, NewLockCataloger(YarnLockCatalogerConfig{})) + TestCataloger(t, NewLockCataloger(CatalogerConfig{})) } @@ -183,7 +183,7 @@ func Test_LockCataloger_Globs(t *testing.T) { pkgtest.NewCatalogTester(). FromDirectory(t, test.fixture). ExpectsResolverContentQueries(test.expected). - TestCataloger(t, NewLockCataloger(YarnLockCatalogerConfig{})) + TestCataloger(t, NewLockCataloger(CatalogerConfig{})) }) } } diff --git a/syft/pkg/cataloger/javascript/options.go b/syft/pkg/cataloger/javascript/options.go index 2bf0078f3a4..cbadaa4ab22 100644 --- a/syft/pkg/cataloger/javascript/options.go +++ b/syft/pkg/cataloger/javascript/options.go @@ -1,18 +1,18 @@ package javascript -type YarnLockCatalogerConfig struct { +type CatalogerConfig struct { searchRemoteLicenses bool } -func (g YarnLockCatalogerConfig) WithSearchRemoteLicenses(input bool) YarnLockCatalogerConfig { +func (g CatalogerConfig) WithSearchRemoteLicenses(input bool) CatalogerConfig { g.searchRemoteLicenses = input return g } // NewCatalogerOpts create a NewCatalogerOpts with default options, which includes: // - searchRemoteLicenses is false -func NewCatalogerOpts() YarnLockCatalogerConfig { - g := YarnLockCatalogerConfig{} +func NewCatalogerOpts() CatalogerConfig { + g := CatalogerConfig{} return g } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index 285d2ac0bf4..11a5780272f 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -13,9 +13,6 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/generic" ) -// integrity check -var _ generic.Parser = parseYarnLock - var ( // packageNameExp matches the name of the dependency in yarn.lock // including scope/namespace prefix if found. @@ -44,10 +41,10 @@ const ( ) type genericYarnLockAdapter struct { - cfg YarnLockCatalogerConfig + cfg CatalogerConfig } -func newGenericYarnLockAdapter(cfg YarnLockCatalogerConfig) genericYarnLockAdapter { +func newGenericYarnLockAdapter(cfg CatalogerConfig) genericYarnLockAdapter { return genericYarnLockAdapter{ cfg: cfg, } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go index cb2dacc407c..78e5099e05a 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go @@ -91,7 +91,8 @@ func TestParseYarnBerry(t *testing.T) { }, } - pkgtest.TestFileParser(t, fixture, parseYarnLock, expectedPkgs, expectedRelationships) + adapter := newGenericYarnLockAdapter(CatalogerConfig{}) + pkgtest.TestFileParser(t, fixture, adapter.parseYarnLock, expectedPkgs, expectedRelationships) } @@ -177,7 +178,8 @@ func TestParseYarnLock(t *testing.T) { }, } - pkgtest.TestFileParser(t, fixture, parseYarnLock, expectedPkgs, expectedRelationships) + adapter := newGenericYarnLockAdapter(CatalogerConfig{}) + pkgtest.TestFileParser(t, fixture, adapter.parseYarnLock, expectedPkgs, expectedRelationships) } From c35310de71975ce661ffa178d44d380f0adc4336 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Tue, 5 Dec 2023 11:37:05 -0500 Subject: [PATCH 4/6] test: test contract for online licenses Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/config.go | 1 + syft/pkg/cataloger/javascript/options.go | 13 +++ syft/pkg/cataloger/javascript/package.go | 24 ++++- .../cataloger/javascript/parse_yarn_lock.go | 6 +- .../javascript/parse_yarn_lock_test.go | 95 ++++++++++++++++++- .../yarn-remote/registry_response.json | 84 ++++++++++++++++ .../test-fixtures/yarn-remote/yarn.lock | 10 ++ 7 files changed, 224 insertions(+), 9 deletions(-) create mode 100644 syft/pkg/cataloger/javascript/test-fixtures/yarn-remote/registry_response.json create mode 100644 syft/pkg/cataloger/javascript/test-fixtures/yarn-remote/yarn.lock diff --git a/syft/pkg/cataloger/config.go b/syft/pkg/cataloger/config.go index f7df5a8f883..7a352e0a17f 100644 --- a/syft/pkg/cataloger/config.go +++ b/syft/pkg/cataloger/config.go @@ -29,6 +29,7 @@ func DefaultConfig() Config { LinuxKernel: kernel.DefaultLinuxCatalogerConfig(), Python: python.DefaultCatalogerConfig(), Java: java.DefaultArchiveCatalogerConfig(), + Javascript: javascript.DefaultCatalogerConfig(), ExcludeBinaryOverlapByOwnership: true, } } diff --git a/syft/pkg/cataloger/javascript/options.go b/syft/pkg/cataloger/javascript/options.go index cbadaa4ab22..61c2cf6f1fd 100644 --- a/syft/pkg/cataloger/javascript/options.go +++ b/syft/pkg/cataloger/javascript/options.go @@ -2,6 +2,7 @@ package javascript type CatalogerConfig struct { searchRemoteLicenses bool + npmBaseURL string } func (g CatalogerConfig) WithSearchRemoteLicenses(input bool) CatalogerConfig { @@ -16,3 +17,15 @@ func NewCatalogerOpts() CatalogerConfig { return g } + +func (g CatalogerConfig) WithNpmBaseURL(input string) CatalogerConfig { + g.npmBaseURL = input + return g +} + +func DefaultCatalogerConfig() CatalogerConfig { + return CatalogerConfig{ + searchRemoteLicenses: false, + npmBaseURL: "https://registry.npmjs.org", + } +} diff --git a/syft/pkg/cataloger/javascript/package.go b/syft/pkg/cataloger/javascript/package.go index aba54898237..eef2ec2d735 100644 --- a/syft/pkg/cataloger/javascript/package.go +++ b/syft/pkg/cataloger/javascript/package.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "net/url" "path" "strings" "time" @@ -109,11 +110,11 @@ func newPnpmPackage(resolver file.Resolver, location file.Location, name, versio ) } -func newYarnLockPackage(searchRemoteLicenses bool, resolver file.Resolver, location file.Location, name, version string) pkg.Package { +func newYarnLockPackage(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name, version string) pkg.Package { var licenseSet pkg.LicenseSet - if searchRemoteLicenses { - license, err := getLicenseFromNpmRegistry(name, version) + if cfg.searchRemoteLicenses { + license, err := getLicenseFromNpmRegistry(cfg.npmBaseURL, name, version) if err == nil && license != "" { licenses := pkg.NewLicensesFromValues(license) licenseSet = pkg.NewLicenseSet(licenses...) @@ -135,8 +136,21 @@ func newYarnLockPackage(searchRemoteLicenses bool, resolver file.Resolver, locat ) } -func getLicenseFromNpmRegistry(packageName, version string) (string, error) { - var requestURL = fmt.Sprintf("https://registry.npmjs.org/%s/%s", packageName, version) +func formatNpmRegistryURL(baseURL, packageName, version string) (requestURL string, err error) { + urlPath := []string{packageName, version} + requestURL, err = url.JoinPath(baseURL, urlPath...) + if err != nil { + return requestURL, fmt.Errorf("unable to format npm request for pkg:version %s%s; %w", packageName, version, err) + } + return requestURL, nil +} + +func getLicenseFromNpmRegistry(basURL, packageName, version string) (string, error) { + // "https://registry.npmjs.org/%s/%s", packageName, version + requestURL, err := formatNpmRegistryURL(basURL, packageName, version) + if err != nil { + return "", fmt.Errorf("unable to format npm request for pkg:version %s%s; %w", packageName, version, err) + } log.Tracef("trying to fetch remote package %s", requestURL) npmRequest, err := http.NewRequest(http.MethodGet, requestURL, nil) diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index 11a5780272f..fdbc51b28bc 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -69,7 +69,7 @@ func (a genericYarnLockAdapter) parseYarnLock(resolver file.Resolver, _ *generic if packageName := findPackageName(line); packageName != noPackage { // When we find a new package, check if we have unsaved identifiers if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) { - pkgs = append(pkgs, newYarnLockPackage(a.cfg.searchRemoteLicenses, resolver, reader.Location, currentPackage, currentVersion)) + pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, currentPackage, currentVersion)) parsedPackages.Add(currentPackage + "@" + currentVersion) } @@ -77,7 +77,7 @@ func (a genericYarnLockAdapter) parseYarnLock(resolver file.Resolver, _ *generic } else if version := findPackageVersion(line); version != noVersion { currentVersion = version } else if packageName, version := findPackageAndVersion(line); packageName != noPackage && version != noVersion && !parsedPackages.Has(packageName+"@"+version) { - pkgs = append(pkgs, newYarnLockPackage(a.cfg.searchRemoteLicenses, resolver, reader.Location, packageName, version)) + pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, packageName, version)) parsedPackages.Add(packageName + "@" + version) // Cleanup to indicate no unsaved identifiers @@ -88,7 +88,7 @@ func (a genericYarnLockAdapter) parseYarnLock(resolver file.Resolver, _ *generic // check if we have valid unsaved data after end-of-file has reached if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) { - pkgs = append(pkgs, newYarnLockPackage(a.cfg.searchRemoteLicenses, resolver, reader.Location, currentPackage, currentVersion)) + pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, currentPackage, currentVersion)) parsedPackages.Add(currentPackage + "@" + currentVersion) } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go index 78e5099e05a..a2fa7c5a34f 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go @@ -1,6 +1,10 @@ package javascript import ( + "io" + "net/http" + "net/http/httptest" + "os" "testing" "github.com/stretchr/testify/assert" @@ -93,7 +97,6 @@ func TestParseYarnBerry(t *testing.T) { adapter := newGenericYarnLockAdapter(CatalogerConfig{}) pkgtest.TestFileParser(t, fixture, adapter.parseYarnLock, expectedPkgs, expectedRelationships) - } func TestParseYarnLock(t *testing.T) { @@ -180,7 +183,60 @@ func TestParseYarnLock(t *testing.T) { adapter := newGenericYarnLockAdapter(CatalogerConfig{}) pkgtest.TestFileParser(t, fixture, adapter.parseYarnLock, expectedPkgs, expectedRelationships) +} +type handlerPath struct { + path string + handler func(w http.ResponseWriter, r *http.Request) +} + +func TestSearchYarnForLicenses(t *testing.T) { + fixture := "test-fixtures/yarn-remote/yarn.lock" + locations := file.NewLocationSet(file.NewLocation(fixture)) + mux, url, teardown := setup() + defer teardown() + tests := []struct { + name string + fixture string + config CatalogerConfig + requestHandlers []handlerPath + expectedPackages []pkg.Package + }{ + { + name: "search remote licenses returns the expected licenses when search is set to true", + config: CatalogerConfig{searchRemoteLicenses: true}, + requestHandlers: []handlerPath{ + { + // https://registry.yarnpkg.com/@babel/code-frame/7.10.4 + path: "/@babel/code-frame/7.10.4", + handler: generateMockNPMHandler("test-fixtures/yarn-remote/registry_response.json"), + }, + }, + expectedPackages: []pkg.Package{ + { + Name: "@babel/code-frame", + Version: "7.10.4", + Locations: locations, + PURL: "pkg:npm/%40babel/code-frame@7.10.4", + Licenses: pkg.NewLicenseSet(pkg.NewLicense("MIT")), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // set up the mock server + for _, handler := range tc.requestHandlers { + mux.HandleFunc(handler.path, handler.handler) + } + tc.config.npmBaseURL = url + adapter := newGenericYarnLockAdapter(tc.config) + pkgtest.TestFileParser(t, fixture, adapter.parseYarnLock, tc.expectedPackages, nil) + }) + } } func TestParseYarnFindPackageNames(t *testing.T) { @@ -338,3 +394,40 @@ func TestParseYarnFindPackageVersions(t *testing.T) { }) } } + +func generateMockNPMHandler(responseFixture string) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + // Copy the file's content to the response writer + file, err := os.Open(responseFixture) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer file.Close() + + _, err = io.Copy(w, file) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +// setup sets up a test HTTP server for mocking requests to maven central. +// The returned url is injected into the Config so the client uses the test server. +// Tests should register handlers on mux to simulate the expected request/response structure +func setup() (mux *http.ServeMux, serverURL string, teardown func()) { + // mux is the HTTP request multiplexer used with the test server. + mux = http.NewServeMux() + + // We want to ensure that tests catch mistakes where the endpoint URL is + // specified as absolute rather than relative. It only makes a difference + // when there's a non-empty base URL path. So, use that. See issue #752. + apiHandler := http.NewServeMux() + apiHandler.Handle("/", mux) + // server is a test HTTP server used to provide mock API responses. + server := httptest.NewServer(apiHandler) + + return mux, server.URL, server.Close +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/yarn-remote/registry_response.json b/syft/pkg/cataloger/javascript/test-fixtures/yarn-remote/registry_response.json new file mode 100644 index 00000000000..3a14eb9b068 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/yarn-remote/registry_response.json @@ -0,0 +1,84 @@ +{ + "name": "@babel/code-frame", + "version": "7.10.4", + "description": "Generate errors that contain a code frame that point to source locations.", + "author": { + "name": "Sebastian McKenzie", + "email": "sebmck@gmail.com" + }, + "homepage": "https://babeljs.io/", + "license": "MIT", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/babel/babel.git", + "directory": "packages/babel-code-frame" + }, + "main": "lib/index.js", + "dependencies": { + "@babel/highlight": "^7.10.4" + }, + "devDependencies": { + "chalk": "^2.0.0", + "strip-ansi": "^4.0.0" + }, + "gitHead": "7fd40d86a0d03ff0e9c3ea16b29689945433d4df", + "bugs": { + "url": "https://github.com/babel/babel/issues" + }, + "_id": "@babel/code-frame@7.10.4", + "_nodeVersion": "14.4.0", + "_npmVersion": "lerna/3.19.0/node@v14.4.0+x64 (darwin)", + "dist": { + "integrity": "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + "shasum": "168da1a36e90da68ae8d49c0f1b48c7c6249213a", + "tarball": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz", + "fileCount": 4, + "unpackedSize": 7723, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJe+zovCRA9TVsSAnZWagAAe8cQAKMtKF7FJx92Re+ol+4B\no0f3yGdzxWUhl8B+e8eXgjYcetMe1MZpG1oQF5ocYo45mZ+ASWMzfp1caT1p\nqt3F5rtADrL9int8ti7ICrWkIGQ3ccnrTxmtx1y9NRGANeARYjlB538xg0xy\n2Gin+2NchK2gxkxeX0nPH7LOcLlJpXecv1p2BK8gFZfabhM4fKfcMSSeljUp\ngzBGU/0CQp3++KsFAS/GMtgo6ZLw7wnHn0IVtDbwhA6A7hpDZar1Q5xsleui\nUTFIJCegGBx9exO1z0fYLGjOuvOdB5790fQnGsspbVTMwpImWpjspmuH1kWI\nTtk1ocnUnvEu1wNK2FMspUeKHNmOi5Jr6bkdGTxecXV4W/p9oEoPe3tHItL5\nbG9gNxY4IUkHeL93D86w9DtgaYtFSCYD6+sY7mQQQdhBrilR06AugBtQG3jP\nFpEsOLSn8vhYOQKv18CN//xaJM/uar40NTfZcQTn4VLXjUsuR4W+3eAv+qb+\nSKEpf8YAhgXJR5EFG1m3m4VHCp40SM1oSibh3/Ib74VZmlF/aq0VcPDs9tD2\no/MibYbQcP01cYxzpfObPXRczTH8TSl3scslcf7aVcLuhbyJtw8gblKnmXjo\njNaZrKjGKrpKRYStMBbCt8ILkL+OZTcDJcihy1x0v4oOWT43lrHnPntOAg7D\nqQbp\r\n=RhVa\r\n-----END PGP SIGNATURE-----\r\n", + "signatures": [ + { + "keyid": "SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA", + "sig": "MEUCIQDSjBYOah3mnIxnAjEKv638MySMCxhZ0J9pexriecmdogIgAOaTRZg3UeVxrs8Khzr78+n4T+10hMn2Z9DRR7k5qEM=" + } + ] + }, + "maintainers": [ + { + "email": "daniel@tschinder.de", + "name": "danez" + }, + { + "email": "bng412@gmail.com", + "name": "existentialism" + }, + { + "email": "hi@henryzoo.com", + "name": "hzoo" + }, + { + "email": "i@jhuang.me", + "name": "jlhwung" + }, + { + "email": "loganfsmyth@gmail.com", + "name": "loganfsmyth" + }, + { + "email": "nicolo.ribaudo@gmail.com", + "name": "nicolo-ribaudo" + } + ], + "_npmUser": { + "name": "jlhwung", + "email": "i@jhuang.me" + }, + "directories": {}, + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/code-frame_7.10.4_1593522734690_0.6416145193889038" + }, + "_hasShrinkwrap": false +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/yarn-remote/yarn.lock b/syft/pkg/cataloger/javascript/test-fixtures/yarn-remote/yarn.lock new file mode 100644 index 00000000000..7c3076d7715 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/yarn-remote/yarn.lock @@ -0,0 +1,10 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4": + version "7.10.4" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.4.tgz#168da1a36e90da68ae8d49c0f1b48c7c6249213a" + integrity sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg== + dependencies: + "@babel/highlight" "^7.10.4" \ No newline at end of file From 91f6f772f48503183be55ea8ea049e7be9404348 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Tue, 5 Dec 2023 11:38:42 -0500 Subject: [PATCH 5/6] log: add warning if failure to fetch when configured Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/javascript/package.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/syft/pkg/cataloger/javascript/package.go b/syft/pkg/cataloger/javascript/package.go index eef2ec2d735..127a44ccf17 100644 --- a/syft/pkg/cataloger/javascript/package.go +++ b/syft/pkg/cataloger/javascript/package.go @@ -119,6 +119,9 @@ func newYarnLockPackage(cfg CatalogerConfig, resolver file.Resolver, location fi licenses := pkg.NewLicensesFromValues(license) licenseSet = pkg.NewLicenseSet(licenses...) } + if err != nil { + log.Warnf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, version, err) + } } return finalizeLockPkg( From 492ab155732456a10a1b8afd82fa34141bb03db2 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Tue, 5 Dec 2023 12:22:11 -0500 Subject: [PATCH 6/6] docs: refactor config to common pattern; update docs Signed-off-by: Christopher Phillips --- README.md | 4 +++ cmd/syft/cli/options/catalog.go | 5 ++-- cmd/syft/cli/options/javascript.go | 3 ++- syft/pkg/cataloger/javascript/config.go | 27 +++++++++++++++++++++ syft/pkg/cataloger/javascript/options.go | 31 ------------------------ 5 files changed, 36 insertions(+), 34 deletions(-) create mode 100644 syft/pkg/cataloger/javascript/config.go delete mode 100644 syft/pkg/cataloger/javascript/options.go diff --git a/README.md b/README.md index bcc8876d64a..1919c357dab 100644 --- a/README.md +++ b/README.md @@ -656,6 +656,10 @@ python: # when given an arbitrary constraint will be used (even if that version may not be available/published). guess-unpinned-requirements: false +javascript: + search-remote-licenses: false + npm-base-url: "https://registry.npmjs.org" + file-contents: cataloger: # enable/disable cataloging of file contents diff --git a/cmd/syft/cli/options/catalog.go b/cmd/syft/cli/options/catalog.go index d05e6092129..dd3999f1f50 100644 --- a/cmd/syft/cli/options/catalog.go +++ b/cmd/syft/cli/options/catalog.go @@ -147,8 +147,9 @@ func (cfg Catalog) ToCatalogerConfig() cataloger.Config { IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives, }, cfg.Java.MaxParentRecursiveDepth), - Javascript: javascriptCataloger.NewCatalogerOpts(). - WithSearchRemoteLicenses(cfg.Javascript.SearchRemoteLicenses), + Javascript: javascriptCataloger.DefaultCatalogerConfig(). + WithSearchRemoteLicenses(cfg.Javascript.SearchRemoteLicenses). + WithNpmBaseURL(cfg.Javascript.NpmBaseURL), Python: pythonCataloger.CatalogerConfig{ GuessUnpinnedRequirements: cfg.Python.GuessUnpinnedRequirements, }, diff --git a/cmd/syft/cli/options/javascript.go b/cmd/syft/cli/options/javascript.go index 094b4a836e2..f3f6383fa30 100644 --- a/cmd/syft/cli/options/javascript.go +++ b/cmd/syft/cli/options/javascript.go @@ -1,5 +1,6 @@ package options type javascript struct { - SearchRemoteLicenses bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"` + SearchRemoteLicenses bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"` + NpmBaseURL string `json:"npm-base-url" yaml:"npm-base-url" mapstructure:"npm-base-url"` } diff --git a/syft/pkg/cataloger/javascript/config.go b/syft/pkg/cataloger/javascript/config.go new file mode 100644 index 00000000000..9c6117970cd --- /dev/null +++ b/syft/pkg/cataloger/javascript/config.go @@ -0,0 +1,27 @@ +package javascript + +const npmBaseURL = "https://registry.npmjs.org" + +type CatalogerConfig struct { + searchRemoteLicenses bool + npmBaseURL string +} + +func DefaultCatalogerConfig() CatalogerConfig { + return CatalogerConfig{ + searchRemoteLicenses: false, + npmBaseURL: npmBaseURL, + } +} + +func (j CatalogerConfig) WithSearchRemoteLicenses(input bool) CatalogerConfig { + j.searchRemoteLicenses = input + return j +} + +func (j CatalogerConfig) WithNpmBaseURL(input string) CatalogerConfig { + if input != "" { + j.npmBaseURL = input + } + return j +} diff --git a/syft/pkg/cataloger/javascript/options.go b/syft/pkg/cataloger/javascript/options.go deleted file mode 100644 index 61c2cf6f1fd..00000000000 --- a/syft/pkg/cataloger/javascript/options.go +++ /dev/null @@ -1,31 +0,0 @@ -package javascript - -type CatalogerConfig struct { - searchRemoteLicenses bool - npmBaseURL string -} - -func (g CatalogerConfig) WithSearchRemoteLicenses(input bool) CatalogerConfig { - g.searchRemoteLicenses = input - return g -} - -// NewCatalogerOpts create a NewCatalogerOpts with default options, which includes: -// - searchRemoteLicenses is false -func NewCatalogerOpts() CatalogerConfig { - g := CatalogerConfig{} - - return g -} - -func (g CatalogerConfig) WithNpmBaseURL(input string) CatalogerConfig { - g.npmBaseURL = input - return g -} - -func DefaultCatalogerConfig() CatalogerConfig { - return CatalogerConfig{ - searchRemoteLicenses: false, - npmBaseURL: "https://registry.npmjs.org", - } -}