Port python cataloger to new generic cataloger pattern

syft/pkg/cataloger/python/cataloger.go

@@ -0,0 +1,26 @@
+package python
+
+import (
+	"github.com/anchore/syft/syft/pkg/cataloger/generic"
+)
+
+const (
+	eggMetadataGlob     = "**/*egg-info/PKG-INFO"
+	eggFileMetadataGlob = "**/*.egg-info"
+	wheelMetadataGlob   = "**/*dist-info/METADATA"
+)
+
+// NewPythonIndexCataloger returns a new cataloger for python packages referenced from poetry lock files, requirements.txt files, and setup.py files.
+func NewPythonIndexCataloger() *generic.Cataloger {
+	return generic.NewCataloger("python-index-cataloger").
+		WithParserByGlobs(parseRequirementsTxt, "**/*requirements*.txt").
+		WithParserByGlobs(parsePoetryLock, "**/poetry.lock").
+		WithParserByGlobs(parsePipfileLock, "**/Pipfile.lock").
+		WithParserByGlobs(parseSetup, "**/setup.py")
+}
+
+// NewPythonPackageCataloger returns a new cataloger for python packages within egg or wheel installation directories.
+func NewPythonPackageCataloger() *generic.Cataloger {
+	return generic.NewCataloger("python-package-cataloger").
+		WithParserByGlobs(parseWheelOrEgg, eggMetadataGlob, eggFileMetadataGlob, wheelMetadataGlob)
+}

syft/pkg/cataloger/python/package_cataloger_test.go → syft/pkg/cataloger/python/cataloger_test.go

@@ -3,13 +3,12 @@ package python
 import (
 	"testing"
 
-	"github.com/go-test/deep"
-
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
 	"github.com/anchore/syft/syft/source"
 )
 
-func TestPythonPackageWheelCataloger(t *testing.T) {
+func Test_PackageCataloger(t *testing.T) {
 	tests := []struct {
 		name            string
 		fixtures        []string
@@ -20,6 +19,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 			fixtures: []string{"test-fixtures/no-version-py3.8.egg-info"},
 			expectedPackage: pkg.Package{
 				Name:         "no-version",
+				PURL:         "pkg:pypi/no-version",
 				Type:         pkg.PythonPkg,
 				Language:     pkg.Python,
 				FoundBy:      "python-package-cataloger",
@@ -40,6 +40,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 			expectedPackage: pkg.Package{
 				Name:         "requests",
 				Version:      "2.22.0",
+				PURL:         "pkg:pypi/requests@2.22.0",
 				Type:         pkg.PythonPkg,
 				Language:     pkg.Python,
 				Licenses:     []string{"Apache 2.0"},
@@ -76,6 +77,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 			expectedPackage: pkg.Package{
 				Name:         "Pygments",
 				Version:      "2.6.1",
+				PURL:         "pkg:pypi/Pygments@2.6.1?vcs_url=git+https://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 				Type:         pkg.PythonPkg,
 				Language:     pkg.Python,
 				Licenses:     []string{"BSD License"},
@@ -112,6 +114,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 			expectedPackage: pkg.Package{
 				Name:         "Pygments",
 				Version:      "2.6.1",
+				PURL:         "pkg:pypi/Pygments@2.6.1",
 				Type:         pkg.PythonPkg,
 				Language:     pkg.Python,
 				Licenses:     []string{"BSD License"},
@@ -142,6 +145,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 			expectedPackage: pkg.Package{
 				Name:         "Pygments",
 				Version:      "2.6.1",
+				PURL:         "pkg:pypi/Pygments@2.6.1",
 				Type:         pkg.PythonPkg,
 				Language:     pkg.Python,
 				Licenses:     []string{"BSD License"},
@@ -164,6 +168,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 			expectedPackage: pkg.Package{
 				Name:         "requests",
 				Version:      "2.22.0",
+				PURL:         "pkg:pypi/requests@2.22.0",
 				Type:         pkg.PythonPkg,
 				Language:     pkg.Python,
 				Licenses:     []string{"Apache 2.0"},
@@ -193,23 +198,15 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 
 			test.expectedPackage.Locations = source.NewLocationSet(locations...)
 
-			actual, _, err := NewPythonPackageCataloger().Catalog(resolver)
-			if err != nil {
-				t.Fatalf("failed to catalog python package: %+v", err)
-			}
-
-			if len(actual) != 1 {
-				t.Fatalf("unexpected number of packages: %d", len(actual))
-			}
-
-			for _, d := range deep.Equal(test.expectedPackage, actual[0]) {
-				t.Errorf("diff: %+v", d)
-			}
+			pkgtest.NewCatalogTester().
+				WithResolver(resolver).
+				Expects([]pkg.Package{test.expectedPackage}, nil).
+				TestCataloger(t, NewPythonPackageCataloger())
 		})
 	}
 }
 
-func TestIgnorePackage(t *testing.T) {
+func Test_PackageCataloger_IgnorePackage(t *testing.T) {
 	tests := []struct {
 		MetadataFixture string
 	}{

syft/pkg/cataloger/python/package.go

@@ -0,0 +1,81 @@
+package python
+
+import (
+	"fmt"
+
+	"github.com/anchore/packageurl-go"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+func newPackageForIndex(name, version string, locations ...source.Location) pkg.Package {
+	p := pkg.Package{
+		Name:      name,
+		Version:   version,
+		Locations: source.NewLocationSet(locations...),
+		PURL:      packageURL(name, version, nil),
+		Language:  pkg.Python,
+		Type:      pkg.PythonPkg,
+	}
+
+	p.SetID()
+
+	return p
+}
+
+func newPackageForPackage(m pkg.PythonPackageMetadata, sources ...source.Location) pkg.Package {
+	var licenses []string
+	if m.License != "" {
+		licenses = []string{m.License}
+	}
+
+	p := pkg.Package{
+		Name:         m.Name,
+		Version:      m.Version,
+		PURL:         packageURL(m.Name, m.Version, &m),
+		Locations:    source.NewLocationSet(sources...),
+		Licenses:     licenses,
+		Language:     pkg.Python,
+		Type:         pkg.PythonPkg,
+		MetadataType: pkg.PythonPackageMetadataType,
+		Metadata:     m,
+	}
+
+	p.SetID()
+	return p
+}
+
+func packageURL(name, version string, m *pkg.PythonPackageMetadata) string {
+	// generate a purl from the package data
+	pURL := packageurl.NewPackageURL(
+		packageurl.TypePyPi,
+		"",
+		name,
+		version,
+		purlQualifiersForPackage(m),
+		"")
+
+	return pURL.ToString()
+}
+
+func purlQualifiersForPackage(m *pkg.PythonPackageMetadata) packageurl.Qualifiers {
+	q := packageurl.Qualifiers{}
+	if m == nil {
+		return q
+	}
+	if m.DirectURLOrigin != nil {
+		q = append(q, vcsURLQualifierForPackage(m.DirectURLOrigin)...)
+	}
+	return q
+}
+
+func vcsURLQualifierForPackage(p *pkg.PythonDirectURLOriginInfo) packageurl.Qualifiers {
+	if p == nil || p.VCS == "" {
+		return nil
+	}
+	// Taken from https://github.com/package-url/purl-spec/blob/master/PURL-SPECIFICATION.rst#known-qualifiers-keyvalue-pairs
+	// packageurl-go still doesn't support all qualifier names
+	return packageurl.Qualifiers{
+		{Key: pkg.PURLQualifierVCSURL, Value: fmt.Sprintf("%s+%s@%s", p.VCS, p.URL, p.CommitID)},
+	}
+}

syft/pkg/cataloger/python/package_test.go

@@ -0,0 +1,46 @@
+package python
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func Test_packageURL(t *testing.T) {
+	tests := []struct {
+		testName string
+		name     string
+		version  string
+		metadata *pkg.PythonPackageMetadata
+		want     string
+	}{
+		{
+			testName: "without metadata",
+			name:     "name",
+			version:  "v0.1.0",
+			want:     "pkg:pypi/name@v0.1.0",
+		},
+		{
+			testName: "with vcs info",
+			name:     "name",
+			version:  "v0.1.0",
+			metadata: &pkg.PythonPackageMetadata{
+				Name:    "bogus",  // note: ignored
+				Version: "v0.2.0", // note: ignored
+				DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{
+					VCS:      "git",
+					URL:      "https://github.com/test/test.git",
+					CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+				},
+			},
+			want: "pkg:pypi/name@v0.1.0?vcs_url=git+https://github.com/test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.testName, func(t *testing.T) {
+			assert.Equal(t, tt.want, packageURL(tt.name, tt.version, tt.metadata))
+		})
+	}
+}

syft/pkg/cataloger/python/parse_pipfile_lock.go

@@ -4,15 +4,15 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"sort"
 	"strings"
 
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/pkg/cataloger/common"
+	"github.com/anchore/syft/syft/pkg/cataloger/generic"
+	"github.com/anchore/syft/syft/source"
 )
 
-type PipfileLock struct {
+type pipfileLock struct {
 	Meta struct {
 		Hash struct {
 			Sha256 string `json:"sha256"`
@@ -35,36 +35,27 @@ type Dependency struct {
 	Version string `json:"version"`
 }
 
-// integrity check
-var _ common.ParserFn = parsePipfileLock
+var _ generic.Parser = parsePipfileLock
 
 // parsePipfileLock is a parser function for Pipfile.lock contents, returning "Default" python packages discovered.
-func parsePipfileLock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
-	packages := make([]*pkg.Package, 0)
+func parsePipfileLock(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
+	pkgs := make([]pkg.Package, 0)
 	dec := json.NewDecoder(reader)
 
 	for {
-		var lock PipfileLock
+		var lock pipfileLock
 		if err := dec.Decode(&lock); err == io.EOF {
 			break
 		} else if err != nil {
 			return nil, nil, fmt.Errorf("failed to parse Pipfile.lock file: %w", err)
 		}
 		for name, pkgMeta := range lock.Default {
 			version := strings.TrimPrefix(pkgMeta.Version, "==")
-			packages = append(packages, &pkg.Package{
-				Name:     name,
-				Version:  version,
-				Language: pkg.Python,
-				Type:     pkg.PythonPkg,
-			})
+			pkgs = append(pkgs, newPackageForIndex(name, version, reader.Location))
 		}
 	}
 
-	// Without sorting the packages slice, the order of packages will be unstable, due to ranging over a map.
-	sort.Slice(packages, func(i, j int) bool {
-		return packages[i].String() < packages[j].String()
-	})
+	pkg.Sort(pkgs)
 
-	return packages, nil, nil
+	return pkgs, nil, nil
 }